diff --git a/CHANGELOG.md b/CHANGELOG.md index eabaa3d0ee..5f0c9f2484 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -90,8 +90,14 @@ ## [0.13.0-rc.3] ### New features - +- Otel rebased to v19 specification version and deprecated v18 features removed. Breaking change. +- GCL, GBQ, G Pub/Sub and Otel based on new tonic/protobuf. No change to GCS ( REST based - goauth in use ) +- Replace `gouth` with `google-authz` for all gRPC/tonic connectivity - refactor GCP gRPC mock testing capability +- Replace `googapis` 0.6 with `google-api-proto` 1.236.0 for all GCP gRPC/tonic connectivity +- Upgrade tonic from 0.6.1 to 0.8.2 -> Common tonic needed across Otel + GCP gRPC connectors +- Upgrade prost from 0.9.0 to 0.11.0 -> HashMap -> BTreeMap for maps/records - Add `dogstatsd` codec for Datadog DogStasD implementation +- Add `compression` field in `otel` connectors for compression support on payloads. ### Fixes @@ -99,6 +105,7 @@ ### Breaking Changes +- Upgrade OpenTelemetry from v0.17 to v0.19 specification compliance - Remove `$elastic._type` field from `elastic` connector response events ## [0.13.0-rc.2] diff --git a/Cargo.lock b/Cargo.lock index 2c71154023..8d20882ec2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "addr2line" -version = "0.17.0" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9ecd88a8c8378ca913a680cd98f0f13ac67383d35993f86c90a70e3f137816b" +checksum = "a76fd60b23679b7d19bd066031410fb7e458ccc5e958eb5c325888ce4baedc97" dependencies = [ "gimli", ] @@ -120,6 +120,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + [[package]] name = "android_system_properties" version = "0.1.5" @@ -213,6 +228,7 @@ version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "942c7cd7ae39e91bde4820d74132e9862e62c2f386c3aa90ccf55949f5bad63a" dependencies = [ + "brotli", "bytes 0.5.6", "flate2", "futures-core", @@ -494,9 +510,9 @@ dependencies = [ "log", "pin-project-lite 0.2.9", "tokio", - "tokio-rustls 0.23.4", + "tokio-rustls", "tungstenite 0.17.3", - "webpki-roots 0.22.5", + "webpki-roots 0.22.6", ] [[package]] @@ -819,7 +835,7 @@ dependencies = [ "pin-project-lite 0.2.9", "pin-utils", "tokio", - "tokio-util 0.7.4", + "tokio-util", "tracing", ] @@ -894,17 +910,63 @@ dependencies = [ "zeroize", ] +[[package]] +name = "axum" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08b108ad2665fa3f6e6a517c3d80ec3e77d224c47d605167aefaa5d7ef97fa48" +dependencies = [ + "async-trait", + "axum-core", + "bitflags", + "bytes 1.3.0", + "futures-util", + "http", + "http-body", + "hyper", + "itoa 1.0.5", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite 0.2.9", + "rustversion", + "serde", + "sync_wrapper", + "tower", + "tower-http", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79b8558f5a0581152dc94dcd289132a1d377494bdeafcd41869b3258e3e2ad92" +dependencies = [ + "async-trait", + "bytes 1.3.0", + "futures-util", + "http", + "http-body", + "mime", + "rustversion", + "tower-layer", + "tower-service", +] + [[package]] name = "backtrace" -version = "0.3.66" +version = "0.3.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cab84319d616cfb654d03394f38ab7e6f0919e181b1b57e1fd15e7fb4077d9a7" +checksum = "233d376d6d185f2a3093e58f283f60f880315b6c60075b01f36b3b85154564ca" dependencies = [ "addr2line", "cc", "cfg-if", "libc", - "miniz_oxide 0.5.4", + "miniz_oxide", "object", "rustc-demangle", ] @@ -1021,6 +1083,27 @@ dependencies = [ "serde_with", ] +[[package]] +name = "brotli" +version = "3.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1a0b1dbcc8ae29329621f8d4f0d835787c1c38bb1401979b49d13b0b305ff68" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ad2d4653bf5ca36ae797b1f4bb4dbddb60ce49ca4aed8a2ce4829f60425b80" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bstr" version = "0.2.17" @@ -1033,6 +1116,16 @@ dependencies = [ "serde", ] +[[package]] +name = "buf_redux" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b953a6887648bb07a535631f2bc00fbdb2a2216f135552cb3f534ed136b9c07f" +dependencies = [ + "memchr", + "safemem", +] + [[package]] name = "bumpalo" version = "3.11.1" @@ -1119,9 +1212,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.0.77" +version = "1.0.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9f73505338f7d905b19d18738976aae232eb46b8efc15554ffc56deb5d9ebe4" +checksum = "a20104e2335ce8a659d6dd92a51a767a0c062599c73b343fd152cb401e828c3d" dependencies = [ "jobserver", ] @@ -1292,7 +1385,7 @@ version = "4.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0177313f9f02afc995627906bbd8967e2be069f5261954222dac78290c2b9014" dependencies = [ - "heck 0.4.0", + "heck", "proc-macro-error", "proc-macro2", "quote", @@ -1729,9 +1822,9 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.82" +version = "1.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4a41a86530d0fe7f5d9ea779916b7cadd2d4f9add748b99c2c029cbbdfaf453" +checksum = "5add3fc1717409d029b20c5b6903fc0c0b02fa6741d820054f4a2efa5e5816fd" dependencies = [ "cc", "cxxbridge-flags", @@ -1741,9 +1834,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.82" +version = "1.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06416d667ff3e3ad2df1cd8cd8afae5da26cf9cec4d0825040f88b5ca659a2f0" +checksum = "b4c87959ba14bc6fbc61df77c3fcfe180fc32b93538c4f1031dd802ccb5f2ff0" dependencies = [ "cc", "codespan-reporting", @@ -1756,15 +1849,15 @@ dependencies = [ [[package]] name = "cxxbridge-flags" -version = "1.0.82" +version = "1.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "820a9a2af1669deeef27cb271f476ffd196a2c4b6731336011e0ba63e2c7cf71" +checksum = "69a3e162fde4e594ed2b07d0f83c6c67b745e7f28ce58c6df5e6b6bef99dfb59" [[package]] name = "cxxbridge-macro" -version = "1.0.82" +version = "1.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a08a6e2fcc370a089ad3b4aaf54db3b1b4cee38ddabce5896b33eb693275f470" +checksum = "3e7e2adeb6a0d4a282e581096b06e1791532b7d576dcde5ccd9382acf55db8e6" dependencies = [ "proc-macro2", "quote", @@ -1832,9 +1925,9 @@ dependencies = [ [[package]] name = "data-encoding" -version = "2.3.2" +version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ee2393c4a91429dffb4bedf19f4d6abf27d8a732c8ce4980305d782e5426d57" +checksum = "23d8666cb01533c39dde32bcbab8e227b4ed6679b2c925eba05feabea39508fb" [[package]] name = "deadpool" @@ -1982,9 +2075,9 @@ checksum = "56899898ce76aaf4a0f24d914c97ea6ed976d42fec6ad33fcbb0a1103e07b2b0" [[package]] name = "dyn-clone" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f94fa09c2aeea5b8839e414b7b841bf429fd25b9c522116ac97ee87856d88b2" +checksum = "c9b0705efd4599c15a38151f4721f7bc388306f61084d3bfd50bd07fbca5cb60" [[package]] name = "either" @@ -2045,7 +2138,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c9720bba047d567ffc8a3cba48bf19126600e249ab7f128e9233e6376976a116" dependencies = [ - "heck 0.4.0", + "heck", "proc-macro2", "quote", "syn", @@ -2121,9 +2214,9 @@ dependencies = [ [[package]] name = "filetime" -version = "0.2.18" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b9663d381d07ae25dc88dbdf27df458faa83a9b25336bcac83d5e452b5fc9d3" +checksum = "4e884668cd0c7480504233e951174ddc3b382f7c2666e3b7310b5c4e7b0c37f9" dependencies = [ "cfg-if", "libc", @@ -2144,7 +2237,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841" dependencies = [ "crc32fast", - "miniz_oxide 0.6.2", + "miniz_oxide", ] [[package]] @@ -2320,6 +2413,21 @@ dependencies = [ "serde_json", ] +[[package]] +name = "gcemeta" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d460327b24cc34c86d53d60a90e9e6044817f7906ebd9baa5c3d0ee13e1ecf" +dependencies = [ + "bytes 1.3.0", + "hyper", + "serde", + "serde_json", + "thiserror", + "tokio", + "tracing", +] + [[package]] name = "generic-array" version = "0.14.6" @@ -2373,9 +2481,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.26.2" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d" +checksum = "dec7af912d60cdbd3677c1af9352ebae6fb8394d165568a2234df0fa00f87793" [[package]] name = "glob" @@ -2420,16 +2528,37 @@ dependencies = [ ] [[package]] -name = "googapis" -version = "0.6.0" +name = "google-api-proto" +version = "1.279.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65e3984a414cce451bd6fa931298c2b384924ddc1b6201adec3b0c0c148dabfa" +checksum = "834323ce60751bae33b9d7ec4a9be26965e3e49c2d6c21ee85f848fa2fea2c65" dependencies = [ "prost", "prost-types", "tonic", ] +[[package]] +name = "google-authz" +version = "1.0.0-alpha.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b957c4d1a775a9f81f74b132ec3705034dff68fd442df015915b66a98984ea1" +dependencies = [ + "bytes 1.3.0", + "futures-util", + "gcemeta 0.2.3", + "hyper", + "hyper-rustls", + "jsonwebtoken 8.2.0", + "parking_lot 0.12.1", + "serde", + "serde_json", + "serde_urlencoded 0.7.1", + "thiserror", + "tower-service", + "tracing", +] + [[package]] name = "gouth" version = "0.2.1" @@ -2437,8 +2566,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c138d157085ba4eb1aaa86e622dc348b956d5ac9d2e446b65941467ebffefdd6" dependencies = [ "attohttpc 0.17.0", - "gcemeta", - "jsonwebtoken", + "gcemeta 0.1.4", + "jsonwebtoken 7.2.0", "serde", "serde_json", "url", @@ -2469,7 +2598,7 @@ dependencies = [ "indexmap", "slab", "tokio", - "tokio-util 0.7.4", + "tokio-util", "tracing", ] @@ -2533,12 +2662,28 @@ dependencies = [ ] [[package]] -name = "heck" -version = "0.3.3" +name = "headers" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3e372db8e5c0d213e0cd0b9be18be2aca3d44cf2fe30a9d46a65581cd454584" +dependencies = [ + "base64 0.13.1", + "bitflags", + "bytes 1.3.0", + "headers-core", + "http", + "httpdate", + "mime", + "sha1 0.10.5", +] + +[[package]] +name = "headers-core" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +checksum = "e7f66481bfee273957b1f20485a4ff3362987f85b2c236580d81b4eb7a326429" dependencies = [ - "unicode-segmentation", + "http", ] [[package]] @@ -2670,6 +2815,12 @@ dependencies = [ "rustls 0.18.1", ] +[[package]] +name = "http-range-header" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29" + [[package]] name = "http-types" version = "2.12.0" @@ -2736,9 +2887,9 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.23.1" +version = "0.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59df7c4e19c950e6e0e868dcc0a300b09a9b88e9ec55bd879ca819087a77355d" +checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c" dependencies = [ "http", "hyper", @@ -2746,7 +2897,7 @@ dependencies = [ "rustls 0.20.7", "rustls-native-certs", "tokio", - "tokio-rustls 0.23.4", + "tokio-rustls", ] [[package]] @@ -2905,9 +3056,18 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.5.1" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11b0d96e660696543b251e58030cf9787df56da39dab19ad60eae7353040917e" + +[[package]] +name = "iri-string" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f88c5561171189e69df9d98bcf18fd5f9558300f7ea7b801eb8a0fd748bd8745" +checksum = "8f0f7638c1e223529f1bfdc48c8b133b9e0b434094d1d28473161ee48b235f78" +dependencies = [ + "nom 7.1.1", +] [[package]] name = "is-terminal" @@ -2967,11 +3127,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "afabcc15e437a6484fc4f12d0fd63068fe457bf93f1c148d3d9649c60b103f32" dependencies = [ "base64 0.12.3", - "pem", + "pem 0.8.3", + "ring", + "serde", + "serde_json", + "simple_asn1 0.4.1", +] + +[[package]] +name = "jsonwebtoken" +version = "8.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f4f04699947111ec1733e71778d763555737579e44b85844cae8e1940a1828" +dependencies = [ + "base64 0.13.1", + "pem 1.1.0", "ring", "serde", "serde_json", - "simple_asn1", + "simple_asn1 0.6.2", ] [[package]] @@ -3119,9 +3293,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.137" +version = "0.2.138" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc7fcc620a3bff7cdd7a365be3376c97191aeaccc2a603e600951e452615bf89" +checksum = "db6d7e329c562c5dfab7a46a2afabc8b987ab9a4834c9d1ca04dc54c1546cef8" [[package]] name = "libflate" @@ -3157,9 +3331,9 @@ dependencies = [ [[package]] name = "link-cplusplus" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9272ab7b96c9046fbc5bc56c06c117cb639fe2d509df0c421cad82d2915cf369" +checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5" dependencies = [ "cc", ] @@ -3172,9 +3346,9 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" [[package]] name = "linux-raw-sys" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f9f08d8963a6c613f4b1a78f4f4a4dbfadf8e6545b2d72861731e4858b8b47f" +checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" [[package]] name = "lock_api" @@ -3290,6 +3464,12 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" +[[package]] +name = "matchit" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40" + [[package]] name = "matrixmultiply" version = "0.3.2" @@ -3345,15 +3525,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" -[[package]] -name = "miniz_oxide" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96590ba8f175222643a85693f33d26e9c8a015f599c216509b1a6894af675d34" -dependencies = [ - "adler", -] - [[package]] name = "miniz_oxide" version = "0.6.2" @@ -3381,6 +3552,24 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" +[[package]] +name = "multipart" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00dec633863867f29cb39df64a397cdf4a6354708ddd7759f70c7fb51c5f9182" +dependencies = [ + "buf_redux", + "httparse", + "log", + "mime", + "mime_guess", + "quick-error 1.2.3", + "rand 0.8.5", + "safemem", + "tempfile", + "twoway", +] + [[package]] name = "native-tls" version = "0.2.11" @@ -3528,9 +3717,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "object" -version = "0.29.0" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21158b2c33aa6d4561f1c0a6ea283ca92bc54802a93b263e910746d679a7eb53" +checksum = "239da7f290cfa979f43f85a8efeee9a8a76d0827c356d37f9d3d7254d6b537fb" dependencies = [ "memchr", ] @@ -3577,9 +3766,9 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "openssl" -version = "0.10.43" +version = "0.10.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "020433887e44c27ff16365eaa2d380547a94544ad509aff6eb5b6e3e0b27b376" +checksum = "b102428fd03bc5edf97f62620f7298614c45cedf287c271e7ed450bbaf83f2e1" dependencies = [ "bitflags", "cfg-if", @@ -3609,9 +3798,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.78" +version = "0.9.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07d5c8cb6e57b3a3612064d7b18b117912b4ce70955c2504d4b741c9e244b132" +checksum = "23bbbf7854cd45b83958ebe919f0e8e516793727652e27fda10a8384cfc790b7" dependencies = [ "autocfg", "cc", @@ -3667,7 +3856,7 @@ checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" dependencies = [ "instant", "lock_api", - "parking_lot_core 0.8.5", + "parking_lot_core 0.8.6", ] [[package]] @@ -3682,9 +3871,9 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" dependencies = [ "cfg-if", "instant", @@ -3729,9 +3918,9 @@ dependencies = [ [[package]] name = "paste" -version = "1.0.9" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1de2e551fb905ac83f73f7aedf2f0cb4a0da7e35efa24a202a936269f1f18e1" +checksum = "d01a5bd0424d00070b0098dd17ebca6f961a959dead1dbcbbbc1d1cd8d3deeba" [[package]] name = "pbkdf2" @@ -3756,6 +3945,15 @@ dependencies = [ "regex", ] +[[package]] +name = "pem" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03c64931a1a212348ec4f3b4362585eca7159d0d09cbdf4a7f74f02173596fd4" +dependencies = [ + "base64 0.13.1", +] + [[package]] name = "percent-encoding" version = "2.2.0" @@ -3900,9 +4098,9 @@ dependencies = [ [[package]] name = "polling" -version = "2.5.1" +version = "2.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "166ca89eb77fd403230b9c156612965a81e094ec6ec3aa13663d4c8b113fa748" +checksum = "22122d5ec4f9fe1b3916419b76be1e80bcb93f618d071d2edf841b137b2a2bd6" dependencies = [ "autocfg", "cfg-if", @@ -3953,6 +4151,16 @@ dependencies = [ "yansi", ] +[[package]] +name = "prettyplease" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c8992a85d8e93a28bdf76137db888d3874e3b230dee5ed8bebac4c9f7617773" +dependencies = [ + "proc-macro2", + "syn", +] + [[package]] name = "proc-macro-crate" version = "1.2.1" @@ -3990,15 +4198,15 @@ dependencies = [ [[package]] name = "proc-macro-hack" -version = "0.5.19" +version = "0.5.20+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" +checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.47" +version = "1.0.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725" +checksum = "57a8eca9f9c4ffde41714334dee777596264c7825420f521abc92b5b5deb63a5" dependencies = [ "unicode-ident", ] @@ -4025,9 +4233,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.9.0" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "444879275cb4fd84958b1a1d5420d15e6fcf7c235fe47f053c9c2a80aceb6001" +checksum = "c0b18e655c21ff5ac2084a5ad0611e827b3f92badf79f4910b5a5c58f4d87ff0" dependencies = [ "bytes 1.3.0", "prost-derive", @@ -4035,29 +4243,31 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.9.0" +version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62941722fb675d463659e49c4f3fe1fe792ff24fe5bbaa9c08cd3b98a1c354f5" +checksum = "276470f7f281b0ed53d2ae42dd52b4a8d08853a3c70e7fe95882acbb98a6ae94" dependencies = [ "bytes 1.3.0", - "heck 0.3.3", + "heck", "itertools", "lazy_static", "log", "multimap", "petgraph", + "prettyplease", "prost", "prost-types", "regex", + "syn", "tempfile", "which", ] [[package]] name = "prost-derive" -version = "0.9.0" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9cc1a3263e07e0bf68e96268f37665207b49560d98739662cdfaae215c720fe" +checksum = "164ae68b6587001ca506d3bf7f1000bfa248d0e1217b618108fba4ec1d0cc306" dependencies = [ "anyhow", "itertools", @@ -4068,9 +4278,9 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.9.0" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "534b7a0e836e3c482d2693070f982e39e7611da9695d4d1f5a4b186b51faef0a" +checksum = "747761bc3dc48f9a34553bf65605cf6cb6288ba219f3450b4275dbd81539551a" dependencies = [ "bytes 1.3.0", "prost", @@ -4096,9 +4306,9 @@ checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" [[package]] name = "quote" -version = "1.0.21" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179" +checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" dependencies = [ "proc-macro2", ] @@ -4200,11 +4410,10 @@ checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" [[package]] name = "rayon" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e060280438193c554f654141c9ea9417886713b7acd75974c85b18a69a88e0b" +checksum = "6db3a213adf02b3bcfd2d3846bb41cb22857d131789e01df434fb7e7bc0759b7" dependencies = [ - "crossbeam-deque", "either", "rayon-core", ] @@ -4329,20 +4538,20 @@ dependencies = [ "pin-project-lite 0.2.9", "rustls 0.20.7", "rustls-native-certs", - "rustls-pemfile", + "rustls-pemfile 1.0.1", "serde", "serde_json", "serde_urlencoded 0.7.1", "tokio", "tokio-native-tls", - "tokio-rustls 0.23.4", - "tokio-util 0.7.4", + "tokio-rustls", + "tokio-util", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "webpki-roots 0.22.5", + "webpki-roots 0.22.6", "winreg", ] @@ -4465,14 +4674,14 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver 1.0.14", + "semver 1.0.16", ] [[package]] name = "rustix" -version = "0.36.4" +version = "0.36.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb93e85278e08bb5788653183213d3a60fc242b10cb9be96586f5a73dcb67c23" +checksum = "a3807b5d10909833d3e9acd1eb5fb988f79376ff10fce42937de71a449c4c588" dependencies = [ "bitflags", "errno", @@ -4527,11 +4736,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0167bac7a9f490495f3c33013e7722b53cb087ecbe082fb0c6387c96f634ea50" dependencies = [ "openssl-probe", - "rustls-pemfile", + "rustls-pemfile 1.0.1", "schannel", "security-framework", ] +[[package]] +name = "rustls-pemfile" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eebeaeb360c87bfb72e84abdb3447159c0eaececf1bef2aecd65a8be949d1c9" +dependencies = [ + "base64 0.13.1", +] + [[package]] name = "rustls-pemfile" version = "1.0.1" @@ -4543,9 +4761,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.9" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97477e48b4cf8603ad5f7aaf897467cf42ab4218a38ef76fb14c2d6773a6d6a8" +checksum = "5583e89e108996506031660fe09baa5011b9dd0341b89029313006d1fb508d70" [[package]] name = "rusty-fork" @@ -4565,6 +4783,12 @@ version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" +[[package]] +name = "safemem" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef703b7cb59335eae2eb93ceb664c0eb7ea6bf567079d843e09420219668e072" + [[package]] name = "same-file" version = "1.0.6" @@ -4584,6 +4808,12 @@ dependencies = [ "windows-sys 0.36.1", ] +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + [[package]] name = "scopeguard" version = "1.1.0" @@ -4592,9 +4822,9 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] name = "scratch" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8132065adcfd6e02db789d9285a0deb2f3fcb04002865ab67d5fb103533898" +checksum = "ddccb15bcce173023b3fedd9436f882a0739b8dfb45e4f6b6002bee5929f61b2" [[package]] name = "sct" @@ -4650,9 +4880,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.14" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e25dfac463d778e353db5be2449d1cce89bd6fd23c9f1ea21310ce6e5a1b29c4" +checksum = "58bc9567378fc7690d6b2addae4e60ac2eeea07becb2c64b9f218b53865cba2a" [[package]] name = "semver-parser" @@ -5002,6 +5232,18 @@ dependencies = [ "num-traits", ] +[[package]] +name = "simple_asn1" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adc4e5204eb1910f40f9cfa375f6f05b68c3abac4b6fd879c8ff5e7ae8a0a085" +dependencies = [ + "num-bigint 0.4.3", + "num-traits", + "thiserror", + "time 0.3.17", +] + [[package]] name = "siphasher" version = "0.3.10" @@ -5232,15 +5474,21 @@ checksum = "45f6ee7c7b87caf59549e9fe45d6a69c75c8019e79e212a835c5da0e92f0ba08" [[package]] name = "syn" -version = "1.0.105" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60b9b43d45702de4c839cb9b51d9f529c5dd26a4aff255b42b1ebc03e88ee908" +checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20518fe4a4c9acf048008599e464deb21beeae3d3578418951a189c235a7a9a8" + [[package]] name = "syslog_loose" version = "0.18.0" @@ -5377,18 +5625,18 @@ checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" [[package]] name = "thiserror" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e" +checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb" +checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f" dependencies = [ "proc-macro2", "quote", @@ -5562,9 +5810,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.22.0" +version = "1.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d76ce4a75fb488c605c54bf610f221cea8b0dafb53333c1a67e8ee199dcd2ae3" +checksum = "eab6d665857cc6ca78d6e80303a02cea7a7851e85dfbd77cbdc09bd129f1ef46" dependencies = [ "autocfg", "bytes 1.3.0", @@ -5572,10 +5820,11 @@ dependencies = [ "memchr", "mio", "num_cpus", + "parking_lot 0.12.1", "pin-project-lite 0.2.9", "socket2", "tokio-macros", - "winapi", + "windows-sys 0.42.0", ] [[package]] @@ -5609,17 +5858,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "tokio-rustls" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6" -dependencies = [ - "rustls 0.19.1", - "tokio", - "webpki 0.21.4", -] - [[package]] name = "tokio-rustls" version = "0.23.4" @@ -5643,17 +5881,15 @@ dependencies = [ ] [[package]] -name = "tokio-util" -version = "0.6.10" +name = "tokio-tungstenite" +version = "0.17.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36943ee01a6d67977dd3f84a5a1d2efeb4ada3a1ae771cadfaa535d9d9fc6507" +checksum = "f714dd15bead90401d77e04243611caec13726c2408afd5b31901dfcdcb3b181" dependencies = [ - "bytes 1.3.0", - "futures-core", - "futures-sink", + "futures-util", "log", - "pin-project-lite 0.2.9", "tokio", + "tungstenite 0.17.3", ] [[package]] @@ -5672,23 +5908,25 @@ dependencies = [ [[package]] name = "toml" -version = "0.5.9" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7" +checksum = "1333c76748e868a4d9d1017b5ab53171dfd095f70c712fdb4653a406547f598f" dependencies = [ "serde", ] [[package]] name = "tonic" -version = "0.6.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff08f4649d10a70ffa3522ca559031285d8e421d727ac85c60825761818f5d0a" +checksum = "8f219fad3b929bef19b1f86fbc0358d35daed8f2cac972037ac0dc10bbb8d5fb" dependencies = [ "async-stream", "async-trait", + "axum", "base64 0.13.1", "bytes 1.3.0", + "flate2", "futures-core", "futures-util", "h2", @@ -5700,23 +5938,26 @@ dependencies = [ "pin-project", "prost", "prost-derive", + "rustls-pemfile 1.0.1", "tokio", - "tokio-rustls 0.22.0", + "tokio-rustls", "tokio-stream", - "tokio-util 0.6.10", + "tokio-util", "tower", "tower-layer", "tower-service", "tracing", "tracing-futures", + "webpki-roots 0.22.6", ] [[package]] name = "tonic-build" -version = "0.6.2" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9403f1bafde247186684b230dc6f38b5cd514584e8bec1dd32514be4745fa757" +checksum = "5bf5e9b9c0f7e0a7c027dcfaba7b2c60816c7049171f679d99ee2ff65d0de8c4" dependencies = [ + "prettyplease", "proc-macro2", "prost-build", "quote", @@ -5750,12 +5991,42 @@ dependencies = [ "rand 0.8.5", "slab", "tokio", - "tokio-util 0.7.4", + "tokio-util", "tower-layer", "tower-service", "tracing", ] +[[package]] +name = "tower-http" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f873044bf02dd1e8239e9c1293ea39dad76dc594ec16185d0a1bf31d8dc8d858" +dependencies = [ + "async-compression", + "base64 0.13.1", + "bitflags", + "bytes 1.3.0", + "futures-core", + "futures-util", + "http", + "http-body", + "http-range-header", + "httpdate", + "iri-string", + "mime", + "mime_guess", + "percent-encoding", + "pin-project-lite 0.2.9", + "tokio", + "tokio-util", + "tower", + "tower-layer", + "tower-service", + "tracing", + "uuid 1.2.2", +] + [[package]] name = "tower-layer" version = "0.3.2" @@ -5849,18 +6120,22 @@ dependencies = [ "globwalk", "halfbrown", "http-types", + "itoa 1.0.5", "lalrpop", + "lexical", "log", "log4rs", "matches", "port_scanner", "pretty_assertions", + "ryu", "serde", "serde_yaml 0.9.16", "shell-words", "signal-hook", "signal-hook-async-std", "simd-json", + "simdutf8", "snmalloc-rs", "surf", "tch", @@ -5911,9 +6186,9 @@ dependencies = [ [[package]] name = "tremor-otelapis" -version = "0.2.4" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c6cfab2469df1b6c86199844e008f549cc6b19ca59ef0b43b68b8a62f21d63b" +checksum = "0f500a42cbb36276e5aa3f8ea439a682517bee79661a77724b5e48343eea8956" dependencies = [ "async-channel", "prost", @@ -5993,7 +6268,8 @@ dependencies = [ "file-mode", "futures", "glob", - "googapis", + "google-api-proto", + "google-authz", "gouth", "grok", "halfbrown", @@ -6005,6 +6281,7 @@ dependencies = [ "http-body", "http-client 6.5.1", "http-types", + "hyper", "indexmap", "itoa 1.0.5", "lazy_static", @@ -6052,6 +6329,8 @@ dependencies = [ "tide", "tide-rustls", "tonic", + "tower", + "tower-http", "tremor-common", "tremor-influx", "tremor-otelapis", @@ -6061,6 +6340,7 @@ dependencies = [ "url", "uuid 1.2.2", "value-trait", + "warp", "xz2", "zstd 0.12.1+zstd.1.5.2", ] @@ -6224,6 +6504,15 @@ dependencies = [ "utf-8", ] +[[package]] +name = "twoway" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59b11b2b5241ba34be09c3cc85a36e56e48f9888862e19cedf23336d35316ed1" +dependencies = [ + "memchr", +] + [[package]] name = "typemap-ors" version = "1.0.0" @@ -6241,9 +6530,9 @@ checksum = "ed5b74f0a24b5454580a79abb6994393b09adf0ab8070f15827cb666255de155" [[package]] name = "typenum" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" +checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" [[package]] name = "uncased" @@ -6271,9 +6560,9 @@ checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" [[package]] name = "unicode-ident" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3" +checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" [[package]] name = "unicode-normalization" @@ -6293,12 +6582,6 @@ dependencies = [ "smallvec", ] -[[package]] -name = "unicode-segmentation" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fdbf052a0783de01e944a6ce7a8cb939e295b1e7be835a1112c3b9a7f047a5a" - [[package]] name = "unicode-width" version = "0.1.10" @@ -6332,9 +6615,9 @@ dependencies = [ [[package]] name = "unsafe-libyaml" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1e5fa573d8ac5f1a856f8d7be41d390ee973daf97c806b2c1a465e4e1406e68" +checksum = "bc7ed8ba44ca06be78ea1ad2c3682a43349126c8818054231ee6f4748012aed2" [[package]] name = "untrusted" @@ -6485,6 +6768,37 @@ dependencies = [ "try-lock", ] +[[package]] +name = "warp" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed7b8be92646fc3d18b06147664ebc5f48d222686cb11a8755e561a735aacc6d" +dependencies = [ + "bytes 1.3.0", + "futures-channel", + "futures-util", + "headers", + "http", + "hyper", + "log", + "mime", + "mime_guess", + "multipart", + "percent-encoding", + "pin-project", + "rustls-pemfile 0.2.1", + "scoped-tls", + "serde", + "serde_json", + "serde_urlencoded 0.7.1", + "tokio", + "tokio-stream", + "tokio-tungstenite", + "tokio-util", + "tower-service", + "tracing", +] + [[package]] name = "wasi" version = "0.9.0+wasi-snapshot-preview1" @@ -6628,9 +6942,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.22.5" +version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368bfe657969fb01238bb756d351dcade285e0f6fcbd36dcb23359a5169975be" +checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87" dependencies = [ "webpki 0.22.0", ] diff --git a/Cargo.toml b/Cargo.toml index 9aee2bbf0d..23f373dd2a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -210,15 +210,12 @@ sled = "0.34" # opentelemetry port_scanner = "0.1.5" -tonic = { version = "0.6.1", default-features = false, features = [ +tonic = { version = "0.8.2", default-features = false, features = [ "transport", "tls", + "tls-webpki-roots", ] } -prost = "0.9.0" -prost-types = "0.9.0" -# This is related to https://github.com/tremor-rs/tremor-runtime/issues/1688 the otel API's need to -# be updated together with tonic -tremor-otelapis = { version = "=0.2.4" } +tremor-otelapis = "0.3.0" # aws-s3 aws-sdk-s3 = "0.21" @@ -227,13 +224,17 @@ aws-config = "0.51" aws-smithy-http = "0.51" # gcp -googapis = { version = "0.6", default-features = false, features = [ +gouth = { version = "0.2" } +google-api-proto = { version = "1.236.0", default-features = false, features = [ "google-pubsub-v1", "google-cloud-bigquery-storage-v1", "google-logging-v2", "google-storage-v2", ] } -gouth = { version = "0.2" } +# gouth = { version = "0.2" } +google-authz = { version = "1.0.0-alpha.4", features = ["tonic"] } +prost = { version = "0.11" } +prost-types = { version = "0.11" } http = "0.2.8" reqwest = { version = "0.11.13", default-features = false, features = [ "rustls-tls", @@ -245,10 +246,16 @@ uuid = { version = "1.2", features = ["v4"] } # wal qwal = { git = "https://github.com/tremor-rs/qwal" } -itoa = "1" -ryu = "1" +warp = "0.3.3" +tower = "0.4.13" +tower-http = { version = "0.3.4", features = [ "full" ] } +hyper = "0.14.20" + +#dogstatsd +simdutf8 = "0.1.4" lexical = "6.1.1" -simdutf8 = "0.1" +itoa = "1.0.5" +ryu = "1.0.12" [dev-dependencies] serial_test = { version = "0.10", features = ["logging"] } @@ -392,4 +399,4 @@ tremor = { path = "/usr/bin/tremor" } "../packaging/distribution/usr/share/tremor/tremor.sh" = { path = "/usr/share/tremor/tremor.sh", mode = "755" } "../tremor-script/lib/" = { path = "/usr/share/tremor/lib/" } # copying systemd service to standard location for rpm packages -"../packaging/distribution/etc/systemd/system/tremor.service" = { path = "/usr/lib/systemd/system/tremor.service" } \ No newline at end of file +"../packaging/distribution/etc/systemd/system/tremor.service" = { path = "/usr/lib/systemd/system/tremor.service" } diff --git a/src/connectors/google.rs b/src/connectors/google.rs index 64a7662db7..a3707c7738 100644 --- a/src/connectors/google.rs +++ b/src/connectors/google.rs @@ -12,288 +12,259 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::{ + fmt::{Display, Formatter}, + task::Poll, +}; + use crate::errors::Result; -use gouth::Token; -use std::sync::Arc; -use std::time::Duration; -use tonic::metadata::MetadataValue; -use tonic::service::Interceptor; -use tonic::{Request, Status}; - -#[async_trait::async_trait] -pub(crate) trait ChannelFactory< - TChannel: tonic::codegen::Service< - http::Request, - Response = http::Response, - > + Clone, -> -{ - async fn make_channel(&self, connect_timeout: Duration) -> Result; +use async_std::pin::Pin; +use futures::future::Either; +use futures::Future; +use futures::FutureExt; +use google_authz::GoogleAuthz; +use http_body::combinators::UnsyncBoxBody; +use http_body::Body; +use tonic::transport::Channel; +use tower::Service; + +#[derive(Debug)] +pub(crate) enum TremorTonicServiceError { + HyperError(hyper::Error), + TonicStatus(tonic::Status), + TonicTransportError(tonic::transport::Error), + GoogleCredentialsError(google_authz::CredentialsError), + GoogleAuthError(google_authz::AuthError), } -pub trait TokenProvider: Clone + Default + Send { - fn get_token(&mut self) -> ::std::result::Result, Status>; +// NOTE Channel is an UnsyncBoxBody internally rather than a tonic::transport::Body +// NOTE tonic::transport::Body is an alias for a hyper::Body +// NOTE All leak into the publcic API for tonic - we wrap the distictions way here +// NOTE to avoid leaking the details of the underlying transport and to enable +// NOTE flexible mock testing of any Google gRPC API that uses the same auth in tremor +type TonicInternalBodyType = UnsyncBoxBody; + +impl Display for TremorTonicServiceError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match *self { + Self::HyperError(ref err) => write!(f, "Hyper error: {err}"), + Self::TonicStatus(ref status) => write!(f, "Tonic error: {status}"), + Self::TonicTransportError(ref err) => write!(f, "Tonic transport error: {err}"), + Self::GoogleAuthError(ref err) => write!(f, "Google Auth error: {err}"), + Self::GoogleCredentialsError(ref err) => write!(f, "Google Credentials error: {err}"), + } + } } -pub struct GouthTokenProvider { - pub(crate) gouth_token: Option, +impl From for TremorTonicServiceError { + fn from(other: hyper::Error) -> Self { + Self::HyperError(other) + } } -impl Clone for GouthTokenProvider { - fn clone(&self) -> Self { - Self { gouth_token: None } +impl From for TremorTonicServiceError { + fn from(other: google_authz::CredentialsError) -> Self { + Self::GoogleCredentialsError(other) } } -impl Default for GouthTokenProvider { - fn default() -> Self { - Self::new() +impl From for TremorTonicServiceError { + fn from(other: google_authz::AuthError) -> Self { + Self::GoogleAuthError(other) } } -impl GouthTokenProvider { - pub fn new() -> Self { - GouthTokenProvider { gouth_token: None } +impl From for TremorTonicServiceError { + fn from(other: tonic::Status) -> Self { + Self::TonicStatus(other) } } -impl TokenProvider for GouthTokenProvider { - fn get_token(&mut self) -> ::std::result::Result, Status> { - let token = if let Some(ref token) = self.gouth_token { - token - } else { - let new_token = - Token::new().map_err(|_| Status::unavailable("Failed to read Google Token"))?; - - self.gouth_token.get_or_insert(new_token) - }; +impl From> for TremorTonicServiceError { + fn from(other: google_authz::Error) -> Self { + match other { + google_authz::Error::Service(err) => Self::TonicTransportError(err), + google_authz::Error::GoogleAuthz(err) => Self::GoogleAuthError(err), + } + } +} - token.header_value().map_err(|e| { - Status::unavailable(format!("Failed to read the Google Token header value: {e}")) - }) +impl From for TremorTonicServiceError { + fn from(other: tonic::transport::Error) -> Self { + Self::TonicTransportError(other) } } +pub(crate) type MockServiceRpcCall = + fn(http::Request) -> http::Response; + #[derive(Clone)] -pub(crate) struct AuthInterceptor -where - T: TokenProvider, -{ - pub token_provider: T, +pub(crate) struct TonicMockService { + delegate: MockServiceRpcCall, } -impl Interceptor for AuthInterceptor -where - T: TokenProvider, -{ - fn call(&mut self, mut request: Request<()>) -> ::std::result::Result, Status> { - let header_value = self.token_provider.get_token()?; - let metadata_value = match MetadataValue::from_str(header_value.as_str()) { - Ok(val) => val, - Err(e) => { - error!("Failed to get token: {}", e); - - return Err(Status::unavailable( - "Failed to retrieve authentication token.", - )); - } - }; - request - .metadata_mut() - .insert("authorization", metadata_value); - - Ok(request) +impl TonicMockService { + pub(crate) fn new(delegate: MockServiceRpcCall) -> Self { + Self { delegate } } } -#[cfg(test)] -#[cfg(feature = "gcp-integration")] -pub(crate) mod tests { - use super::*; - use crate::{ - connectors::utils::EnvHelper, - errors::{Error, Result}, - }; - use std::io::Write; - - #[derive(Clone)] - pub struct TestTokenProvider { - token: Arc, +impl std::error::Error for TremorTonicServiceError {} + +impl Service> for TonicMockService { + type Response = http::Response; + type Error = TremorTonicServiceError; + type Future = Pin< + Box< + dyn Future< + Output = std::result::Result< + http::Response, + TremorTonicServiceError, + >, + > + Send, + >, + >; + + fn poll_ready( + &mut self, + _cx: &mut std::task::Context<'_>, + ) -> Poll> { + Poll::Ready(Ok(())) } - impl Default for TestTokenProvider { - fn default() -> Self { - Self::new() - } + fn call(&mut self, request: http::Request) -> Self::Future { + let response = (self.delegate)(request); + futures::future::ok(response).boxed() } +} - impl TestTokenProvider { - pub fn new() -> Self { - Self { - token: Arc::new(String::new()), - } - } +#[derive(Clone)] +pub(crate) enum TremorGoogleAuthz { + Code(GoogleAuthz), + Test(TonicMockService), +} - pub fn new_with_token(token: Arc) -> Self { - Self { token } - } +// Hoist google-authz errors so we have a common service error whether +// mock testing or in production code for gRPC GCP connectors. +fn map_poll_err( + result: Poll>>, +) -> Poll> { + match result { + Poll::Ready(Err(google_authz::Error::GoogleAuthz(err))) => Poll::Ready(Err(err.into())), + Poll::Ready(Err(err @ google_authz::Error::Service(_))) => Poll::Ready(Err(err.into())), + Poll::Ready(Ok(value)) => Poll::Ready(Ok(value)), + Poll::Pending => Poll::Pending, } +} - impl TokenProvider for TestTokenProvider { - fn get_token(&mut self) -> ::std::result::Result, Status> { - Ok(self.token.clone()) +type TonicResponseFuture = futures::future::MapErr< + tonic::transport::channel::ResponseFuture, + fn(tonic::transport::Error) -> google_authz::Error, +>; +type HttpResponseFuture = std::future::Ready< + std::result::Result< + http::Response, + google_authz::Error, + >, +>; +type NormalizedResponse = + std::result::Result, TremorTonicServiceError>; + +async fn map_ready_err( + result: Either, +) -> NormalizedResponse { + match result.await { + Err(google_authz::Error::GoogleAuthz(err)) => Err(err.into()), + Err(err @ google_authz::Error::Service(_)) => Err(err.into()), + Ok(response) => { + let (_parts, body) = response.into_parts(); + let body = hyper::body::to_bytes(body).await?; + let body = http_body::Full::new(body); + let body = http_body::combinators::BoxBody::new(body).map_err(|err| match err {}); + let body = tonic::body::BoxBody::new(body); + Ok(http::Response::new(body)) } } +} - #[derive(serde::Serialize, serde::Deserialize)] - struct ServiceAccount { - client_email: String, - private_key_id: String, - private_key: String, - token_uri: String, +impl Service> for TremorGoogleAuthz { + type Response = http::Response; + type Error = TremorTonicServiceError; + type Future = Pin< + Box< + dyn Future< + Output = std::result::Result< + http::Response, + TremorTonicServiceError, + >, + > + Send, + >, + >; + + fn poll_ready( + &mut self, + cx: &mut std::task::Context<'_>, + ) -> Poll> { + match self { + Self::Code(authz) => map_poll_err(authz.poll_ready(cx)), + Self::Test(authz) => authz.poll_ready(cx), + } } - #[derive(serde::Serialize, serde::Deserialize)] - struct TokenResponse { - token_type: String, - access_token: String, - expires_in: u64, + fn call(&mut self, req: http::Request) -> Self::Future { + match self { + Self::Code(authz) => { + let result = authz.call(req); + Box::pin(map_ready_err(result)) + } + Self::Test(authz) => authz.call(req), + } } +} - /// Some random generated private key that isn't used anywhere else - const PRIVATE_KEY: &str = "-----BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC/SZoFm3528gDJ -vMQBeTGm6dohSfqstFoYYVtGEDGnt9GwkjbJcnIAIiON+Qw7wV5v24UFJKQ8Eg/q -Jf8bF0PT6yvSW+cof/94OgGz/PyPwrHVGniEy2Wbe1qYkDaQfxDzyPP5hKetmoof -FF8u1IyJYdduxBm80eYG/JVYhn85ycV4zVUWPzuF7BmBmK4n1DX8HlD3qQWtVtiP -DCQ1H7pKSn6nDLlQtv6zEx5gnfnVIC/G2hB414FqTxkwLI5ae5njOeh9aFzTzD5Y -hifcPqjs91fJ4tO4/VfesyrOWOowAIil7ZaWNd6CsljiC0iqt15oohBKbFz/wGSv -DxTiavvRAgMBAAECggEAAT9Rd/IxLPhItu5z7ovthE7eK2oZ1OjFKKEKSq0eDpLe -7p8sqJVTA65O6ItXjNRm0WU1tOU6nyJBnjXnhLP0lYWhG5Lm8W23Cv/n1TzHIdUN -bbWpoQYMttEv87KgpHV4dRQaB5LzOMLUxHCdauCbo2UZSRSrk7HG5ZDdx9eMR1Wg -vkhk3S70dyheO804BwSkvpxCbjcgg2ILRn5EacL0uU7GNxGQUCInNK2LTN0gUSKg -qLITAE2CE0cwcs6DzPgHk3M78AlTILDYbKmOIB3FPImTY88crR9OMvqDbraKTvwb -sS2M5gWOO0LDOeXVuIxG9j0J3hxxSY6aGHJRt+d5BQKBgQDLQ3Ri6OXirtd2gxZv -FY65lHQd+LMrWO2R31zv2aif+XoJRh5PXM5kN5Cz6eFp/z2E5DWa1ubF4nPSBc3S -fW96LGwnBLOIOccxJ6wdfLY+sw/U2PEDhUP5Z0NxHr4x0AOxfQTrEmnSyx6oE04Q -rXtqpiCg8pP+za6Hx1ZWFx1YxQKBgQDw6rbv+Wadz+bnuOYWyy7GUv7ZXVWup1gU -IoZgR5h6ZMNyFpK2NlzLOctzttkWdoV9fn4ux6T3kBWrJdbd9WkCGom2SX6b3PqH -evcZ73RvbuHVjtm9nHov9eqU+pcz8Se3NZVEhsov1FWboBE5E+i1qO0jiOaJRFEm -aIlaK9gPnQKBgDkmx0PETlb1aDm3VAh53D6L4jZHJkGK6Il6b0w1O/d3EvwmjgEs -jA+bnAEqQqomDSsfa38U66A6MuybmyqTAFQux14VMVGdRUep6vgDh86LVGk5clLW -Fq26fjkBNuMUpOUzzL032S9e00jY3LtNvATZnxUB/+DF/kvJHZppN2QtAoGAB/7S -KW6ugChJMoGJaVI+8CgK+y3EzTISk0B+Ey3tGorDjcLABboSJFB7txBnbf5q+bo7 -99N6XxjyDycHVYByhrZYwar4v7V6vwpOrxaqV5RnfE3sXgWWbIcNzPnwELI9LjBi -Ds8mYKX8XVjXmXxWqci8bgR6Gi4hP1QS0uJHnmUCgYEAiDbOiUed1gL1yogrTG4w -r+S/aL2pt/xBNz9Dw+cZnqnZHWDuewU8UCO6mrupM8MXEAfRnzxyUX8b7Yk/AoFo -sEUlZGvHmBh8nBk/7LJVlVcVRWQeQ1kg6b+m6thwRz6HsKIvExpNYbVkzqxbeJW3 -PX8efvDMhv16QqDFF0k80d0= ------END PRIVATE KEY-----"; - - #[async_std::test] - async fn gouth_token() -> Result<()> { - let mut file = tempfile::NamedTempFile::new()?; - - let port = crate::connectors::tests::free_port::find_free_tcp_port().await?; - let sa = ServiceAccount { - client_email: "snot@tremor.rs".to_string(), - private_key_id: "badger".to_string(), - private_key: PRIVATE_KEY.to_string(), - token_uri: format!("http://127.0.0.1:{port}/"), - }; - let sa_str = simd_json::serde::to_string_pretty(&sa)?; - file.as_file_mut().write_all(sa_str.as_bytes())?; - let path = file.into_temp_path(); - let path_str = path.to_string_lossy().to_string(); - let mut env = EnvHelper::new(); - env.set_var("GOOGLE_APPLICATION_CREDENTIALS", &path_str); - - let mut provider = GouthTokenProvider::default(); - assert!(provider.get_token().is_err()); - - let mut server = tide::new(); - server.at("/").post(|_| async { - Ok(simd_json::serde::to_string_pretty(&TokenResponse { - token_type: "snot".to_string(), - access_token: "access_token".to_string(), - expires_in: 100_000_000, - })?) - }); - let server_handle = async_std::task::spawn(async move { - server.listen(format!("127.0.0.1:{port}")).await?; - Ok::<(), Error>(()) - }); - let token = provider.get_token()?; - assert_eq!(token.as_str(), "snot access_token"); - - server_handle.cancel().await; - - // token is cached, no need to call again - let token = provider.get_token()?; - assert_eq!(token.as_str(), "snot access_token"); - - Ok(()) +impl TremorGoogleAuthz { + // Create a new live or production channel to a GCP gRPC service. + pub async fn new(transport: Channel) -> Result { + let transport_channel = GoogleAuthz::new(transport).await; + Ok(Self::Code(transport_channel)) } - #[test] - fn appease_the_coverage_gods() { - let provider = GouthTokenProvider::default(); - let mut provider = provider; - assert!(provider.get_token().is_err()); - - let provider = FailingTokenProvider::default(); - let mut provider = provider; - assert!(provider.get_token().is_err()); + // Create a new mock channel to a GCP gRPC service + pub fn new_mock(logic: MockServiceRpcCall) -> Self { + Self::Test(TonicMockService::new(logic)) } +} - #[test] - fn interceptor_can_add_the_auth_header() -> Result<()> { - let mut interceptor = AuthInterceptor { - token_provider: TestTokenProvider::new_with_token(Arc::new("test".to_string())), - }; - let request = Request::new(()); - - let result = interceptor.call(request)?; - - assert!(result - .metadata() - .get("authorization") - .map(|m| m == "test") - .unwrap_or_default()); - Ok(()) - } +#[cfg(test)] +pub(crate) mod tests { - #[derive(Clone, Default)] - struct FailingTokenProvider {} + use super::*; - impl TokenProvider for FailingTokenProvider { - fn get_token(&mut self) -> std::result::Result, Status> { - Err(Status::unavailable("boo")) - } + fn fake_body(content: Vec) -> TonicInternalBodyType { + let body = bytes::Bytes::from(content); + let body = http_body::Full::new(body); + let body = http_body::combinators::BoxBody::new(body).map_err(|err| match err {}); + tonic::body::BoxBody::new(body) } - #[test] - fn interceptor_will_pass_token_error() { - let mut interceptor = AuthInterceptor { - token_provider: FailingTokenProvider {}, - }; - let request = Request::new(()); - - let result = interceptor.call(request); - - assert!(result.is_err()); + fn empty_body() -> TonicInternalBodyType { + fake_body(vec![]) } - #[test] - fn interceptor_fails_on_invalid_token_value() { - let mut interceptor = AuthInterceptor { - // control characters (ASCII < 32) are not allowed - token_provider: TestTokenProvider::new_with_token(Arc::new("\r\n".into())), - }; - let request = Request::new(()); - - let result = interceptor.call(request); - - assert!(result.is_err()); + // NOTE We have a public github based CI/CD pipeline for non cloud provider specific protocols + // NOTE and we have a CNCF equinix env for benchmarks and cloud provider agnostic tests + // NOTE so we use mock tests where emulators/simulators are not available at this time + #[async_std::test] + async fn appease_the_coverage_gods() -> Result<()> { + let mut mock = TremorGoogleAuthz::new_mock(|_| http::Response::new(empty_body())); + let actual = mock.call(http::Request::new(empty_body())).await; + if let Ok(actual) = actual { + let actual = hyper::body::to_bytes(actual).await?; + let expected = hyper::body::to_bytes(empty_body()).await?; + assert_eq!(actual, expected); + } else { + return Err("snot".into()); + } + Ok(()) } } diff --git a/src/connectors/impls/gbq/writer.rs b/src/connectors/impls/gbq/writer.rs index 8b4659a0c3..e15cf074ef 100644 --- a/src/connectors/impls/gbq/writer.rs +++ b/src/connectors/impls/gbq/writer.rs @@ -14,8 +14,7 @@ mod sink; -use crate::connectors::google::GouthTokenProvider; -use crate::connectors::impls::gbq::writer::sink::{GbqSink, TonicChannelFactory}; +use crate::connectors::impls::gbq::writer::sink::GbqSink; use crate::connectors::prelude::*; use crate::connectors::{Connector, ConnectorBuilder, ConnectorConfig, ConnectorType}; use serde::Deserialize; @@ -26,15 +25,15 @@ pub(crate) struct Config { pub table_id: String, pub connect_timeout: u64, pub request_timeout: u64, - #[serde(default = "default_request_size_limit")] - pub request_size_limit: usize, + // #[serde(default = "default_request_size_limit")] + // pub request_size_limit: usize, } impl ConfigImpl for Config {} -fn default_request_size_limit() -> usize { - // 10MB - 10 * 1024 * 1024 -} +// fn default_request_size_limit() -> usize { +// // 10MB +// 10 * 1024 * 1024 +// } #[derive(Debug, Default)] pub(crate) struct Builder {} @@ -50,10 +49,7 @@ impl Connector for Gbq { sink_context: SinkContext, builder: SinkManagerBuilder, ) -> Result> { - let sink = GbqSink::::new( - self.config.clone(), - Box::new(TonicChannelFactory), - ); + let sink = GbqSink::new(self.config.clone()); builder.spawn(sink, sink_context).map(Some) } @@ -97,7 +93,7 @@ mod tests { table_id: "test".into(), connect_timeout: 1, request_timeout: 1, - request_size_limit: 10 * 1024 * 1024, + // request_size_limit: 10 * 1024 * 1024, }, }; diff --git a/src/connectors/impls/gbq/writer/sink.rs b/src/connectors/impls/gbq/writer/sink.rs index 71494502ea..a4274d0410 100644 --- a/src/connectors/impls/gbq/writer/sink.rs +++ b/src/connectors/impls/gbq/writer/sink.rs @@ -12,68 +12,37 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::connectors::google::ChannelFactory; -use crate::connectors::{ - google::{AuthInterceptor, TokenProvider}, - impls::gbq::writer::Config, - prelude::*, -}; -use async_std::prelude::{FutureExt, StreamExt}; +use crate::connectors::google::TremorGoogleAuthz; +use crate::connectors::impls::gbq::writer::Config; +use crate::connectors::prelude::*; +use async_std::prelude::FutureExt; +use async_std::stream::StreamExt; +use bytes::Bytes; use futures::stream; -use googapis::google::cloud::bigquery::storage::v1::append_rows_request::Rows; -use googapis::google::cloud::bigquery::storage::v1::{ - append_rows_request::{self, ProtoData}, - append_rows_response::{AppendResult, Response}, - big_query_write_client::BigQueryWriteClient, - table_field_schema::{self, Type as TableType}, - write_stream, AppendRowsRequest, CreateWriteStreamRequest, ProtoRows, ProtoSchema, - TableFieldSchema, WriteStream, +use google_api_proto::google::cloud::bigquery::storage::v1::append_rows_request::ProtoData; +use google_api_proto::google::cloud::bigquery::storage::v1::big_query_write_client::BigQueryWriteClient; +use google_api_proto::google::cloud::bigquery::storage::v1::table_field_schema::Type as TableType; +use google_api_proto::google::cloud::bigquery::storage::v1::write_stream::WriteMode; +use google_api_proto::google::cloud::bigquery::storage::v1::{ + append_rows_request, table_field_schema, write_stream, AppendRowsRequest, + CreateWriteStreamRequest, ProtoRows, ProtoSchema, TableFieldSchema, WriteStream, }; use prost::encoding::WireType; -use prost::Message; use prost_types::{field_descriptor_proto, DescriptorProto, FieldDescriptorProto}; -use std::collections::hash_map::Entry; -use std::marker::PhantomData; -use std::{collections::HashMap, time::Duration}; -use tonic::{ - codegen::InterceptedService, - transport::{Certificate, Channel, ClientTlsConfig}, -}; - -pub(crate) struct TonicChannelFactory; - -#[async_trait::async_trait] -impl ChannelFactory for TonicChannelFactory { - async fn make_channel(&self, connect_timeout: Duration) -> Result { - let tls_config = ClientTlsConfig::new() - .ca_certificate(Certificate::from_pem(googapis::CERTIFICATES)) - .domain_name("bigquerystorage.googleapis.com"); - - Ok( - Channel::from_static("https://bigquerystorage.googleapis.com") - .connect_timeout(connect_timeout) - .tls_config(tls_config)? - .connect() - .await?, - ) - } -} +use std::collections::HashMap; +use std::time::Duration; +use tonic::transport::{Channel, ClientTlsConfig}; -struct ConnectedWriteStream { - name: String, - mapping: JsonToProtobufMapping, -} +// +// NOTE This code now depends on a different protocol buffers and tonic library +// that in turn introduce a different GCP auth mechanism +// -pub(crate) struct GbqSink< - T: TokenProvider, - TChannel: GbqChannel, - TChannelError: GbqChannelError, -> { - client: Option>>>, - write_streams: HashMap, +pub(crate) struct GbqSink { + client: Option>, + write_stream: Option, + mapping: Option, config: Config, - channel_factory: Box + Send + Sync>, - _error_phantom: PhantomData, } struct Field { @@ -199,6 +168,8 @@ fn map_field( fn encode_field(val: &Value, field: &Field, result: &mut Vec) -> Result<()> { let tag = field.tag; + // fixme check which fields are required and fail if they're missing + // fixme do not panic if the tremor type does not match match field.table_type { TableType::Double => prost::encoding::double::encode( tag, @@ -292,7 +263,7 @@ impl JsonToProtobufMapping { } } - pub fn map(&self, value: &Value) -> Result> { + pub fn map(&self, value: &Value) -> Result { if let Some(obj) = value.as_object() { let mut result = Vec::with_capacity(obj.len()); @@ -302,7 +273,7 @@ impl JsonToProtobufMapping { } } - return Ok(result); + return Ok(result.into()); } Err(ErrorKind::BigQueryTypeMismatch("object", value.value_type()).into()) @@ -312,32 +283,19 @@ impl JsonToProtobufMapping { &self.descriptor } } -impl, TChannelError: GbqChannelError> - GbqSink -{ - pub fn new( - config: Config, - channel_factory: Box + Send + Sync>, - ) -> Self { +impl GbqSink { + pub fn new(config: Config) -> Self { Self { client: None, - write_streams: HashMap::new(), + write_stream: None, + mapping: None, config, - channel_factory, - _error_phantom: PhantomData, } } } #[async_trait::async_trait] -impl< - T: TokenProvider + 'static, - TChannel: GbqChannel + 'static, - TChannelError: GbqChannelError, - > Sink for GbqSink -where - TChannel::Future: Send, -{ +impl Sink for GbqSink { async fn on_event( &mut self, _input: &str, @@ -346,110 +304,125 @@ where _serializer: &mut EventSerializer, _start: u64, ) -> Result { - let request_size_limit = self.config.request_size_limit; - - let request_data = self - .event_to_requests(event, ctx, request_size_limit) - .await?; - - if request_data.len() > 1 { - warn!("{ctx} The batch is too large to be sent in a single request, splitting it into {} requests. Consider lowering the batch size.", request_data.len()); - } - let client = self.client.as_mut().ok_or(ErrorKind::ClientNotAvailable( "BigQuery", "The client is not connected", ))?; + let write_stream = self + .write_stream + .as_ref() + .ok_or(ErrorKind::ClientNotAvailable( + "BigQuery", + "The write stream is not available", + ))?; + let mapping = self.mapping.as_mut().ok_or(ErrorKind::ClientNotAvailable( + "BigQuery", + "The mapping is not available", + ))?; - for request in request_data { - let req_timeout = Duration::from_nanos(self.config.request_timeout); - let append_response = client - .append_rows(stream::iter(vec![request])) - .timeout(req_timeout) - .await; + let mut serialized_rows = Vec::with_capacity(event.len()); - let append_response = if let Ok(append_response) = append_response { - append_response - } else { - // timeout sending append rows request - error!( - "{ctx} GBQ request timed out after {}ms", - req_timeout.as_millis() - ); - ctx.notifier.connection_lost().await?; - - return Ok(SinkReply::FAIL); - }; + for data in event.value_iter() { + serialized_rows.push(mapping.map(data)?); + } - if let Ok(x) = append_response? - .into_inner() - .next() - .timeout(req_timeout) - .await - { - match x { - Some(Ok(res)) => { - if let Some(updated_schema) = res.updated_schema.as_ref() { - let fields = updated_schema - .fields - .iter() - .map(|f| { - format!( - "{}: {:?}", - f.name, - TableType::from_i32(f.r#type).unwrap_or_default() - ) - }) - .collect::>() - .join("\n"); - - info!("{ctx} GBQ Schema was updated: {}", fields); - } - if let Some(res) = res.response { - match res { - Response::AppendResult(AppendResult { .. }) => {} - Response::Error(e) => { - error!("{ctx} GBQ Error: {} {}", e.code, e.message); - return Ok(SinkReply::FAIL); - } - } - } - } - Some(Err(e)) => { - error!("{ctx} GBQ Error: {}", e); - return Ok(SinkReply::FAIL); - } - None => return Ok(SinkReply::NONE), + let request = AppendRowsRequest { + write_stream: write_stream.name.clone(), + offset: None, + trace_id: String::new(), + rows: Some(append_rows_request::Rows::ProtoRows(ProtoData { + writer_schema: Some(ProtoSchema { + proto_descriptor: Some(mapping.descriptor().clone()), + }), + rows: Some(ProtoRows { serialized_rows }), + })), + }; + let req_timeout = Duration::from_nanos(self.config.request_timeout); + let append_response = client + .append_rows(stream::iter(vec![request])) + .timeout(req_timeout) + .await; + + let append_response = if let Ok(append_response) = append_response { + append_response + } else { + // timeout sending append rows request + error!( + "{ctx} GBQ request timed out after {}ms", + req_timeout.as_millis() + ); + ctx.notifier.connection_lost().await?; + + return Ok(SinkReply::FAIL); + }; + if let Ok(x) = append_response? + .into_inner() + .next() + .timeout(req_timeout) + .await + { + match x { + Some(Ok(_)) => Ok(SinkReply::ACK), + Some(Err(e)) => { + error!("{ctx} GBQ Error: {}", e); + Ok(SinkReply::FAIL) } - } else { - // timeout receiving response - error!( - "{ctx} Receiving GBQ response timeout after {}ms", - req_timeout.as_millis() - ); - ctx.notifier.connection_lost().await?; - - return Ok(SinkReply::FAIL); + None => Ok(SinkReply::NONE), } - } + } else { + // timeout receiving response + error!( + "{ctx} Receiving GBQ response timeout after {}ms", + req_timeout.as_millis() + ); + ctx.notifier.connection_lost().await?; - Ok(SinkReply::ACK) + Ok(SinkReply::FAIL) + } } async fn connect(&mut self, ctx: &SinkContext, _attempt: &Attempt) -> Result { info!("{ctx} Connecting to BigQuery"); - let channel = self - .channel_factory - .make_channel(Duration::from_nanos(self.config.connect_timeout)) + let tls_config = ClientTlsConfig::new().domain_name("bigquerystorage.googleapis.com"); + + let channel = Channel::from_static("https://bigquerystorage.googleapis.com") + .connect_timeout(Duration::from_nanos(self.config.connect_timeout)) + .tls_config(tls_config)? + .connect() .await?; - let client = BigQueryWriteClient::with_interceptor( - channel, - AuthInterceptor { - token_provider: T::default(), - }, + let mut client = BigQueryWriteClient::new(TremorGoogleAuthz::new(channel).await?); + + let write_stream = client + .create_write_stream(CreateWriteStreamRequest { + parent: self.config.table_id.clone(), + write_stream: Some(WriteStream { + // The stream name here will be ignored and a generated value will be set in the response + name: String::new(), + r#type: i32::from(write_stream::Type::Committed), + create_time: None, + commit_time: None, + table_schema: None, + location: String::new(), // Should be a valid region TODO FIXME + write_mode: WriteMode::Insert.into(), + }), + }) + .await? + .into_inner(); + + let mapping = JsonToProtobufMapping::new( + &write_stream + .table_schema + .as_ref() + .ok_or(ErrorKind::GbqSinkFailed("Table schema was not provided"))? + .clone() + .fields, + ctx, ); + + self.mapping = Some(mapping); + self.write_stream = Some(write_stream); self.client = Some(client); Ok(true) @@ -460,331 +433,15 @@ where } } -pub trait GbqChannel: - tonic::codegen::Service< - http::Request, - Response = http::Response, - Error = TChannelError, - > + Send - + Clone -where - TChannelError: GbqChannelError, -{ -} - -pub trait GbqChannelError: - Into> + Send + Sync -{ -} - -impl GbqChannelError for T where - T: Into> + Send + Sync -{ -} -impl GbqChannel for T -where - T: tonic::codegen::Service< - http::Request, - Response = http::Response, - Error = TChannelError, - > + Send - + Clone, - TChannelError: GbqChannelError, -{ -} - -impl GbqSink -where - T: TokenProvider + 'static, - TChannel: GbqChannel + 'static, - TChannel::Future: Send, - TChannelError: GbqChannelError, -{ - async fn event_to_requests( - &mut self, - event: Event, - ctx: &SinkContext, - request_size_limit: usize, - ) -> Result> { - let mut request_data: Vec = Vec::new(); - let mut requests: HashMap = HashMap::new(); - - for (data, meta) in event.value_meta_iter() { - let write_stream = self - .get_or_create_write_stream( - &ctx.extract_meta(meta) - .get("table_id") - .as_str() - .map_or_else(|| self.config.table_id.clone(), ToString::to_string), - ctx, - ) - .await?; - - match requests.entry(write_stream.name.clone()) { - Entry::Occupied(entry) => { - let mut request = entry.remove(); - - let serialized_event = write_stream.mapping.map(data)?; - Self::rows_from_request(&mut request)?.push(serialized_event); - - if request.encoded_len() > request_size_limit { - let rows = Self::rows_from_request(&mut request)?; - let row_count = rows.len(); - let last_event = rows.pop().ok_or(ErrorKind::GbqSinkFailed( - "Failed to pop last event from request", - ))?; - request_data.push(request); - - let mut new_rows = Vec::with_capacity(event.len() - row_count); - new_rows.push(last_event); - - requests.insert( - write_stream.name.clone(), - AppendRowsRequest { - write_stream: write_stream.name.clone(), - offset: None, - rows: Some(append_rows_request::Rows::ProtoRows(ProtoData { - writer_schema: Some(ProtoSchema { - proto_descriptor: Some( - write_stream.mapping.descriptor().clone(), - ), - }), - rows: Some(ProtoRows { - serialized_rows: new_rows, - }), - })), - trace_id: String::new(), - }, - ); - } - } - Entry::Vacant(entry) => { - let serialized_event = write_stream.mapping.map(data)?; - let mut serialized_rows = Vec::with_capacity(event.len()); - serialized_rows.push(serialized_event); - - entry.insert(AppendRowsRequest { - write_stream: write_stream.name.clone(), - offset: None, - rows: Some(append_rows_request::Rows::ProtoRows(ProtoData { - writer_schema: Some(ProtoSchema { - proto_descriptor: Some(write_stream.mapping.descriptor().clone()), - }), - rows: Some(ProtoRows { serialized_rows }), - })), - trace_id: String::new(), - }); - } - } - } - - for (_, request) in requests { - request_data.push(request); - } - - Ok(request_data) - } -} - -impl< - T: TokenProvider + 'static, - TChannel: GbqChannel + 'static, - TChannelError: GbqChannelError, - > GbqSink -where - TChannel::Future: Send, -{ - async fn get_or_create_write_stream( - &mut self, - table_id: &str, - ctx: &SinkContext, - ) -> Result<&ConnectedWriteStream> { - let client = self.client.as_mut().ok_or(ErrorKind::ClientNotAvailable( - "BigQuery", - "The client is not connected", - ))?; - - match self.write_streams.entry(table_id.to_string()) { - Entry::Occupied(entry) => { - // NOTE: `into_mut` is needed here, even though we just need a non-mutable reference - // This is because `get` returns reference which's lifetime is bound to the entry, - // while the reference returned by `into_mut` is bound to the map - Ok(entry.into_mut()) - } - Entry::Vacant(entry) => { - let stream = client - .create_write_stream(CreateWriteStreamRequest { - parent: table_id.to_string(), - write_stream: Some(WriteStream { - // The stream name here will be ignored and a generated value will be set in the response - name: String::new(), - r#type: i32::from(write_stream::Type::Committed), - create_time: None, - commit_time: None, - table_schema: None, - }), - }) - .await? - .into_inner(); - - let mapping = JsonToProtobufMapping::new( - &stream - .table_schema - .as_ref() - .ok_or_else(|| ErrorKind::GbqSchemaNotProvided(table_id.to_string()))? - .clone() - .fields, - ctx, - ); - - Ok(entry.insert(ConnectedWriteStream { - name: stream.name, - mapping, - })) - } - } - } -} - -impl< - T: TokenProvider + 'static, - TChannel: GbqChannel + 'static, - TChannelError: GbqChannelError, - > GbqSink -where - TChannel::Future: Send, -{ - fn rows_from_request(request: &mut AppendRowsRequest) -> Result<&mut Vec>> { - let rows = match request - .rows - .as_mut() - .ok_or(ErrorKind::GbqSinkFailed("No rows in request"))? - { - Rows::ProtoRows(ref mut x) => { - &mut x - .rows - .as_mut() - .ok_or(ErrorKind::GbqSinkFailed("No rows in request"))? - .serialized_rows - } - }; - - Ok(rows) - } -} - #[cfg(test)] -#[cfg(feature = "gcp-integration")] mod test { use super::*; use crate::connectors::impls::gbq; use crate::connectors::reconnect::ConnectionLostNotifier; use crate::connectors::tests::ConnectorHarness; - use crate::connectors::{ - google::tests::TestTokenProvider, utils::quiescence::QuiescenceBeacon, - }; - use bytes::Bytes; - use futures::future::Ready; - use googapis::google::cloud::bigquery::storage::v1::table_field_schema::Mode; - use googapis::google::cloud::bigquery::storage::v1::{ - append_rows_response, AppendRowsResponse, TableSchema, - }; - use googapis::google::rpc::Status; - use http::{HeaderMap, HeaderValue}; - use prost::Message; - use std::collections::VecDeque; - use std::fmt::{Display, Formatter}; - use std::sync::{Arc, RwLock}; - use std::task::Poll; - use tonic::body::BoxBody; - use tonic::codegen::Service; - use tremor_common::ids::SinkId; + use google_api_proto::google::cloud::bigquery::storage::v1::table_field_schema::Mode; use value_trait::StaticNode; - struct HardcodedChannelFactory { - channel: Channel, - } - - #[async_trait::async_trait] - impl ChannelFactory for HardcodedChannelFactory { - async fn make_channel(&self, _connect_timeout: Duration) -> Result { - Ok(self.channel.clone()) - } - } - - #[derive(Debug)] - enum MockServiceError {} - - impl Display for MockServiceError { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "MockServiceError") - } - } - - impl std::error::Error for MockServiceError {} - - struct MockChannelFactory { - responses: Arc>>>, - } - - #[async_trait::async_trait] - impl ChannelFactory for MockChannelFactory { - async fn make_channel(&self, _connect_timeout: Duration) -> Result { - Ok(MockService { - responses: self.responses.clone(), - }) - } - } - - #[derive(Clone)] - struct MockService { - responses: Arc>>>, - } - - impl Service> for MockService { - type Response = http::Response; - type Error = MockServiceError; - type Future = - Ready, MockServiceError>>; - - fn poll_ready( - &mut self, - _cx: &mut std::task::Context<'_>, - ) -> Poll> { - Poll::Ready(Ok(())) - } - - #[allow(clippy::unwrap_used, clippy::cast_possible_truncation)] // We don't control the return type here - fn call(&mut self, _request: http::Request) -> Self::Future { - let buffer = self.responses.write().unwrap().pop_front().unwrap(); - - let (mut tx, body) = tonic::transport::Body::channel(); - let jh = async_std::task::spawn(async move { - let len: [u8; 4] = (buffer.len() as u32).to_be_bytes(); - - let mut response_buffer = vec![0u8]; - response_buffer.append(&mut len.to_vec()); - response_buffer.append(&mut buffer.clone()); - - tx.send_data(Bytes::from(response_buffer)).await.unwrap(); - - let mut trailers = HeaderMap::new(); - trailers.insert( - "content-type", - HeaderValue::from_static("application/grpc+proto"), - ); - trailers.insert("grpc-status", HeaderValue::from_static("0")); - - tx.send_trailers(trailers).await.unwrap(); - }); - async_std::task::spawn_blocking(|| jh); - - let response = http::Response::new(body); - - futures::future::ready(Ok(response)) - } - } - #[test] fn skips_unknown_field_types() { let (rx, _tx) = async_std::channel::unbounded(); @@ -796,16 +453,16 @@ mod test { r#type: -1, mode: Mode::Required.into(), fields: vec![], - description: String::new(), + description: "".to_string(), max_length: 0, precision: 0, scale: 0, }], &SinkContext { - uid: SinkId::default(), + uid: Default::default(), alias: Alias::new("flow", "connector"), - connector_type: ConnectorType::default(), - quiescence_beacon: QuiescenceBeacon::default(), + connector_type: Default::default(), + quiescence_beacon: Default::default(), notifier: ConnectionLostNotifier::new(rx), }, ); @@ -825,16 +482,16 @@ mod test { r#type: TableType::Unspecified.into(), mode: Mode::Required.into(), fields: vec![], - description: String::new(), + description: "".to_string(), max_length: 0, precision: 0, scale: 0, }], &SinkContext { - uid: SinkId::default(), + uid: Default::default(), alias: Alias::new("flow", "connector"), - connector_type: ConnectorType::default(), - quiescence_beacon: QuiescenceBeacon::default(), + connector_type: Default::default(), + quiescence_beacon: Default::default(), notifier: ConnectionLostNotifier::new(rx), }, ); @@ -863,23 +520,23 @@ mod test { r#type: item.0.into(), mode: Mode::Required.into(), fields: vec![], - description: String::new(), + description: "".to_string(), max_length: 0, precision: 0, scale: 0, }], &SinkContext { - uid: SinkId::default(), + uid: Default::default(), alias: Alias::new("flow", "connector"), - connector_type: ConnectorType::default(), - quiescence_beacon: QuiescenceBeacon::default(), + connector_type: Default::default(), + quiescence_beacon: Default::default(), notifier: ConnectionLostNotifier::new(rx), }, ); assert_eq!(result.1.len(), 1); assert_eq!(result.1["something"].table_type, item.0); - assert_eq!(result.0.field[0].r#type, Some(item.1.into())); + assert_eq!(result.0.field[0].r#type, Some(item.1.into())) } } @@ -898,21 +555,21 @@ mod test { r#type: TableType::Int64.into(), mode: Mode::Required.into(), fields: vec![], - description: String::new(), + description: "".to_string(), max_length: 0, precision: 0, scale: 0, }], - description: String::new(), + description: "".to_string(), max_length: 0, precision: 0, scale: 0, }], &SinkContext { - uid: SinkId::default(), + uid: Default::default(), alias: Alias::new("flow", "connector"), - connector_type: ConnectorType::default(), - quiescence_beacon: QuiescenceBeacon::default(), + connector_type: Default::default(), + quiescence_beacon: Default::default(), notifier: ConnectionLostNotifier::new(rx), }, ); @@ -927,7 +584,7 @@ mod test { assert_eq!( result.1["something"].subfields["subfield_a"].table_type, TableType::Int64 - ); + ) } #[test] @@ -938,7 +595,7 @@ mod test { Field { table_type: TableType::Int64, tag: 1, - subfields: HashMap::default(), + subfields: Default::default(), }, ), ( @@ -946,7 +603,7 @@ mod test { Field { table_type: TableType::String, tag: 2, - subfields: HashMap::default(), + subfields: Default::default(), }, ), ]; @@ -961,7 +618,7 @@ mod test { } #[test] - pub fn can_encode_stringy_types() { + pub fn test_can_encode_stringy_types() { // NOTE: This test always passes the string "I" as the value to encode, this is not correct for some of the types (e.g. datetime), // but we still allow it, leaving the validation to BigQuery let data = [ @@ -983,7 +640,7 @@ mod test { &Field { table_type: item, tag: 123, - subfields: HashMap::default() + subfields: Default::default() }, &mut result ) @@ -997,7 +654,7 @@ mod test { } #[test] - pub fn can_encode_a_struct() { + pub fn test_can_encode_a_struct() { let mut values = halfbrown::HashMap::new(); values.insert("a".into(), Value::Static(StaticNode::I64(1))); values.insert("b".into(), Value::Static(StaticNode::I64(1024))); @@ -1009,7 +666,7 @@ mod test { Field { table_type: TableType::Int64, tag: 1, - subfields: HashMap::default(), + subfields: Default::default(), }, ); subfields.insert( @@ -1017,7 +674,7 @@ mod test { Field { table_type: TableType::Int64, tag: 2, - subfields: HashMap::default(), + subfields: Default::default(), }, ); @@ -1030,7 +687,7 @@ mod test { let mut result = Vec::new(); assert!(encode_field(&input, &field, &mut result).is_ok()); - assert_eq!([130u8, 64u8, 5u8, 8u8, 1u8, 16u8, 128u8, 8u8], result[..]); + assert_eq!([130u8, 64u8, 5u8, 8u8, 1u8, 16u8, 128u8, 8u8], result[..]) } #[test] @@ -1039,7 +696,7 @@ mod test { let field = Field { table_type: TableType::Double, tag: 2, - subfields: HashMap::default(), + subfields: Default::default(), }; let mut result = Vec::new(); @@ -1057,7 +714,7 @@ mod test { let field = Field { table_type: TableType::Bool, tag: 43, - subfields: HashMap::default(), + subfields: Default::default(), }; let mut result = Vec::new(); @@ -1072,7 +729,7 @@ mod test { let field = Field { table_type: TableType::Bytes, tag: 1, - subfields: HashMap::default(), + subfields: Default::default(), }; let mut result = Vec::new(); @@ -1087,7 +744,7 @@ mod test { let field = Field { table_type: TableType::Json, tag: 1, - subfields: HashMap::default(), + subfields: Default::default(), }; let mut result = Vec::new(); @@ -1103,7 +760,7 @@ mod test { let field = Field { table_type: TableType::Interval, tag: 1, - subfields: HashMap::default(), + subfields: Default::default(), }; let mut result = Vec::new(); @@ -1119,7 +776,7 @@ mod test { let field = Field { table_type: TableType::Unspecified, tag: 1, - subfields: HashMap::default(), + subfields: Default::default(), }; let mut result = Vec::new(); @@ -1134,10 +791,10 @@ mod test { let (rx, _tx) = async_std::channel::unbounded(); let sink_context = SinkContext { - uid: SinkId::default(), + uid: Default::default(), alias: Alias::new("flow", "connector"), - connector_type: ConnectorType::default(), - quiescence_beacon: QuiescenceBeacon::default(), + connector_type: Default::default(), + quiescence_beacon: Default::default(), notifier: ConnectionLostNotifier::new(rx), }; let mapping = JsonToProtobufMapping::new( @@ -1147,7 +804,7 @@ mod test { r#type: TableType::Int64.into(), mode: Mode::Required.into(), fields: vec![], - description: String::new(), + description: "".to_string(), max_length: 0, precision: 0, scale: 0, @@ -1157,7 +814,7 @@ mod test { r#type: TableType::Int64.into(), mode: Mode::Required.into(), fields: vec![], - description: String::new(), + description: "".to_string(), max_length: 0, precision: 0, scale: 0, @@ -1169,24 +826,24 @@ mod test { let descriptor = mapping.descriptor(); assert_eq!(2, descriptor.field.len()); assert_eq!( - descriptor.field[0].r#type, - Some(field_descriptor_proto::Type::Int64 as i32), + field_descriptor_proto::Type::Int64 as i32, + descriptor.field[0].r#type.unwrap() ); assert_eq!( - descriptor.field[1].r#type, - Some(field_descriptor_proto::Type::Int64 as i32), + field_descriptor_proto::Type::Int64 as i32, + descriptor.field[1].r#type.unwrap() ); } #[test] - pub fn can_map_json_to_protobuf() -> Result<()> { + pub fn can_map_json_to_protobuf() { let (rx, _tx) = async_std::channel::unbounded(); let sink_context = SinkContext { - uid: SinkId::default(), + uid: Default::default(), alias: Alias::new("flow", "connector"), - connector_type: ConnectorType::default(), - quiescence_beacon: QuiescenceBeacon::default(), + connector_type: Default::default(), + quiescence_beacon: Default::default(), notifier: ConnectionLostNotifier::new(rx), }; let mapping = JsonToProtobufMapping::new( @@ -1196,7 +853,7 @@ mod test { r#type: TableType::Int64.into(), mode: Mode::Required.into(), fields: vec![], - description: String::new(), + description: "".to_string(), max_length: 0, precision: 0, scale: 0, @@ -1206,7 +863,7 @@ mod test { r#type: TableType::Int64.into(), mode: Mode::Required.into(), fields: vec![], - description: String::new(), + description: "".to_string(), max_length: 0, precision: 0, scale: 0, @@ -1217,21 +874,20 @@ mod test { let mut fields = halfbrown::HashMap::new(); fields.insert("a".into(), Value::Static(StaticNode::I64(12))); fields.insert("b".into(), Value::Static(StaticNode::I64(21))); - let result = mapping.map(&Value::Object(Box::new(fields)))?; + let result = mapping.map(&Value::Object(Box::new(fields))).unwrap(); assert_eq!([8u8, 12u8, 16u8, 21u8], result[..]); - Ok(()) } #[test] - fn map_field_ignores_fields_that_are_not_in_definition() -> Result<()> { + fn map_field_ignores_fields_that_are_not_in_definition() { let (rx, _tx) = async_std::channel::unbounded(); let sink_context = SinkContext { - uid: SinkId::default(), + uid: Default::default(), alias: Alias::new("flow", "connector"), - connector_type: ConnectorType::default(), - quiescence_beacon: QuiescenceBeacon::default(), + connector_type: Default::default(), + quiescence_beacon: Default::default(), notifier: ConnectionLostNotifier::new(rx), }; let mapping = JsonToProtobufMapping::new( @@ -1241,7 +897,7 @@ mod test { r#type: TableType::Int64.into(), mode: Mode::Required.into(), fields: vec![], - description: String::new(), + description: "".to_string(), max_length: 0, precision: 0, scale: 0, @@ -1251,7 +907,7 @@ mod test { r#type: TableType::Int64.into(), mode: Mode::Required.into(), fields: vec![], - description: String::new(), + description: "".to_string(), max_length: 0, precision: 0, scale: 0, @@ -1263,21 +919,20 @@ mod test { fields.insert("a".into(), Value::Static(StaticNode::I64(12))); fields.insert("b".into(), Value::Static(StaticNode::I64(21))); fields.insert("c".into(), Value::Static(StaticNode::I64(33))); - let result = mapping.map(&Value::Object(Box::new(fields)))?; + let result = mapping.map(&Value::Object(Box::new(fields))).unwrap(); assert_eq!([8u8, 12u8, 16u8, 21u8], result[..]); - Ok(()) } #[test] - fn map_field_ignores_struct_fields_that_are_not_in_definition() -> Result<()> { + fn map_field_ignores_struct_fields_that_are_not_in_definition() { let (rx, _tx) = async_std::channel::unbounded(); let sink_context = SinkContext { - uid: SinkId::default(), + uid: Default::default(), alias: Alias::new("flow", "connector"), - connector_type: ConnectorType::default(), - quiescence_beacon: QuiescenceBeacon::default(), + connector_type: Default::default(), + quiescence_beacon: Default::default(), notifier: ConnectionLostNotifier::new(rx), }; let mapping = JsonToProtobufMapping::new( @@ -1290,12 +945,12 @@ mod test { r#type: TableType::Int64.into(), mode: Mode::Required.into(), fields: vec![], - description: String::new(), + description: "".to_string(), max_length: 0, precision: 0, scale: 0, }], - description: String::new(), + description: "".to_string(), max_length: 0, precision: 0, scale: 0, @@ -1307,10 +962,9 @@ mod test { inner_fields.insert("y".into(), Value::Static(StaticNode::I64(10))); let mut fields = halfbrown::HashMap::new(); fields.insert("a".into(), Value::Object(Box::new(inner_fields))); - let result = mapping.map(&Value::Object(Box::new(fields)))?; + let result = mapping.map(&Value::Object(Box::new(fields))).unwrap(); assert_eq!([10u8, 2u8, 8u8, 10u8], result[..]); - Ok(()) } #[test] @@ -1318,10 +972,10 @@ mod test { let (rx, _tx) = async_std::channel::unbounded(); let sink_context = SinkContext { - uid: SinkId::default(), + uid: Default::default(), alias: Alias::new("flow", "connector"), - connector_type: ConnectorType::default(), - quiescence_beacon: QuiescenceBeacon::default(), + connector_type: Default::default(), + quiescence_beacon: Default::default(), notifier: ConnectionLostNotifier::new(rx), }; let mapping = JsonToProtobufMapping::new( @@ -1330,7 +984,7 @@ mod test { r#type: TableType::Bytes.into(), mode: Mode::Required.into(), fields: vec![], - description: String::new(), + description: "".to_string(), max_length: 0, precision: 0, scale: 0, @@ -1344,7 +998,7 @@ mod test { if let Err(Error(ErrorKind::BigQueryTypeMismatch("bytes", x), _)) = result { assert_eq!(x, ValueType::I64); } else { - panic!("Bytes conversion did not fail on type mismatch"); + assert!(false, "Bytes conversion did not fail on type mismatch"); } } @@ -1353,10 +1007,10 @@ mod test { let (rx, _tx) = async_std::channel::unbounded(); let sink_context = SinkContext { - uid: SinkId::default(), + uid: Default::default(), alias: Alias::new("flow", "connector"), - connector_type: ConnectorType::default(), - quiescence_beacon: QuiescenceBeacon::default(), + connector_type: Default::default(), + quiescence_beacon: Default::default(), notifier: ConnectionLostNotifier::new(rx), }; let mapping = JsonToProtobufMapping::new( @@ -1365,7 +1019,7 @@ mod test { r#type: TableType::Bytes.into(), mode: Mode::Required.into(), fields: vec![], - description: String::new(), + description: "".to_string(), max_length: 0, precision: 0, scale: 0, @@ -1377,7 +1031,7 @@ mod test { if let Err(Error(ErrorKind::BigQueryTypeMismatch("object", x), _)) = result { assert_eq!(x, ValueType::I64); } else { - panic!("Mapping did not fail on non-object event"); + assert!(false, "Mapping did not fail on non-object event"); } } @@ -1397,26 +1051,26 @@ mod test { } #[async_std::test] - async fn on_event_fails_if_client_is_not_conected() -> Result<()> { + async fn on_event_fails_if_client_is_not_connected() -> Result<()> { let (rx, _tx) = async_std::channel::unbounded(); let config = Config::new(&literal!({ "table_id": "doesnotmatter", - "connect_timeout": 1_000_000, - "request_timeout": 1_000_000 - }))?; + "connect_timeout": 1000000, + "request_timeout": 1000000 + })) + .unwrap(); - let mut sink = - GbqSink::::new(config, Box::new(TonicChannelFactory)); + let mut sink = GbqSink::new(config); let result = sink .on_event( "", Event::signal_tick(), &SinkContext { - uid: SinkId::default(), + uid: Default::default(), alias: Alias::new("flow", "connector"), - connector_type: ConnectorType::default(), - quiescence_beacon: QuiescenceBeacon::default(), + connector_type: Default::default(), + quiescence_beacon: Default::default(), notifier: ConnectionLostNotifier::new(rx), }, &mut EventSerializer::new( @@ -1425,49 +1079,8 @@ mod test { vec![], &ConnectorType::from(""), &Alias::new("flow", "connector"), - )?, - 0, - ) - .await; - - assert!(result.is_err()); - Ok(()) - } - - #[async_std::test] - async fn on_event_fails_if_write_stream_is_not_conected() -> Result<()> { - let (rx, _tx) = async_std::channel::unbounded(); - let config = Config::new(&literal!({ - "table_id": "doesnotmatter", - "connect_timeout": 1_000_000, - "request_timeout": 1_000_000 - }))?; - - let mut sink = GbqSink::::new( - config, - Box::new(HardcodedChannelFactory { - channel: Channel::from_static("http://example.com").connect_lazy(), - }), - ); - - let result = sink - .on_event( - "", - Event::signal_tick(), - &SinkContext { - uid: SinkId::default(), - alias: Alias::new("flow", "connector"), - connector_type: ConnectorType::default(), - quiescence_beacon: QuiescenceBeacon::default(), - notifier: ConnectionLostNotifier::new(rx), - }, - &mut EventSerializer::new( - None, - CodecReq::Structured, - vec![], - &ConnectorType::from(""), - &Alias::new("flow", "connector"), - )?, + ) + .unwrap(), 0, ) .await; @@ -1475,210 +1088,4 @@ mod test { assert!(result.is_err()); Ok(()) } - - #[async_std::test] - pub async fn fails_on_error_response() -> Result<()> { - let mut buffer_write_stream = vec![]; - let mut buffer_append_rows_response = vec![]; - WriteStream { - name: "test".to_string(), - r#type: i32::from(write_stream::Type::Committed), - create_time: None, - commit_time: None, - table_schema: Some(TableSchema { fields: vec![] }), - } - .encode(&mut buffer_write_stream) - .map_err(|_| "encode failed")?; - - AppendRowsResponse { - updated_schema: Some(TableSchema { - fields: vec![TableFieldSchema { - name: "newfield".to_string(), - r#type: i32::from(table_field_schema::Type::String), - mode: i32::from(Mode::Nullable), - fields: vec![], - description: "test".to_string(), - max_length: 10, - precision: 0, - scale: 0, - }], - }), - response: Some(append_rows_response::Response::Error(Status { - code: 1024, - message: "test failure".to_string(), - details: vec![], - })), - } - .encode(&mut buffer_append_rows_response) - .map_err(|_| "encode failed")?; - - let mut sink = GbqSink::::new( - Config { - table_id: String::new(), - connect_timeout: 1_000_000_000, - request_timeout: 1_000_000_000, - request_size_limit: 10 * 1024 * 1024, - }, - Box::new(MockChannelFactory { - responses: Arc::new(RwLock::new(VecDeque::from([ - buffer_write_stream, - buffer_append_rows_response, - ]))), - }), - ); - - let sink_context = SinkContext { - uid: SinkId::default(), - alias: Alias::new("flow", "connector"), - connector_type: ConnectorType::default(), - quiescence_beacon: QuiescenceBeacon::default(), - notifier: ConnectionLostNotifier::new(async_std::channel::unbounded().0), - }; - - sink.connect(&sink_context, &Attempt::default()).await?; - - let result = sink - .on_event( - "", - Event::default(), - &sink_context, - &mut EventSerializer::new( - None, - CodecReq::Structured, - vec![], - &ConnectorType::from(""), - &Alias::new("flow", "connector"), - )?, - 0, - ) - .await?; - - assert_eq!(result.ack, SinkAck::Fail); - assert_eq!(result.cb, CbAction::None); - Ok(()) - } - - #[async_std::test] - pub async fn splits_large_requests() -> Result<()> { - let mut buffer_write_stream = vec![]; - let mut buffer_append_rows_response = vec![]; - WriteStream { - name: "test".to_string(), - r#type: i32::from(write_stream::Type::Committed), - create_time: None, - commit_time: None, - table_schema: Some(TableSchema { - fields: vec![TableFieldSchema { - name: "a".to_string(), - r#type: table_field_schema::Type::String as i32, - mode: table_field_schema::Mode::Nullable as i32, - fields: vec![], - description: String::new(), - max_length: 0, - precision: 0, - scale: 0, - }], - }), - } - .encode(&mut buffer_write_stream) - .map_err(|_| "encode failed")?; - - AppendRowsResponse { - updated_schema: None, - response: Some(append_rows_response::Response::AppendResult(AppendResult { - offset: None, - })), - } - .encode(&mut buffer_append_rows_response) - .map_err(|_| "encode failed")?; - - let responses = Arc::new(RwLock::new(VecDeque::from([ - buffer_write_stream, - buffer_append_rows_response.clone(), - buffer_append_rows_response, - ]))); - let mut sink = GbqSink::::new( - Config { - table_id: String::new(), - connect_timeout: 1_000_000_000, - request_timeout: 1_000_000_000, - request_size_limit: 16 * 1024, - }, - Box::new(MockChannelFactory { - responses: responses.clone(), - }), - ); - - let sink_context = SinkContext { - uid: SinkId::default(), - alias: Alias::new("flow", "connector"), - connector_type: ConnectorType::default(), - quiescence_beacon: QuiescenceBeacon::default(), - notifier: ConnectionLostNotifier::new(async_std::channel::unbounded().0), - }; - - sink.connect(&sink_context, &Attempt::default()).await?; - - let value = literal!([ - { - "data": { - "value": { - "a": "a".repeat(15*1024) - }, - "meta": {} - } - }, - { - "data": { - "value": { - "a": "b".repeat(15*1024) - }, - "meta": {} - } - } - ]); - - let payload: EventPayload = value.into(); - - let result = sink - .on_event( - "", - Event { - data: payload, - is_batch: true, - ..Default::default() - }, - &sink_context, - &mut EventSerializer::new( - None, - CodecReq::Structured, - vec![], - &ConnectorType::from(""), - &Alias::new("flow", "connector"), - )?, - 0, - ) - .await?; - - assert_eq!(result.ack, SinkAck::Ack); - assert_eq!(0, responses.read()?.len()); - Ok(()) - } - - #[async_std::test] - pub async fn does_not_auto_ack() { - let sink = GbqSink::::new( - Config { - table_id: String::new(), - connect_timeout: 1_000_000_000, - request_timeout: 1_000_000_000, - request_size_limit: 10 * 1024 * 1024, - }, - Box::new(MockChannelFactory { - responses: Arc::new(RwLock::new(VecDeque::new())), - }), - ); - - assert!(!sink.auto_ack()); - } } diff --git a/src/connectors/impls/gcl/writer.rs b/src/connectors/impls/gcl/writer.rs index 051220c02a..394cd48da5 100644 --- a/src/connectors/impls/gcl/writer.rs +++ b/src/connectors/impls/gcl/writer.rs @@ -15,17 +15,15 @@ pub(crate) mod meta; mod sink; -use crate::connectors::google::GouthTokenProvider; -use crate::connectors::impls::gcl::writer::sink::{GclSink, TonicChannelFactory}; +use crate::connectors::impls::gcl::writer::sink::GclSink; use crate::connectors::prelude::*; use crate::connectors::{Alias, Connector, ConnectorBuilder, ConnectorConfig, ConnectorType}; use crate::errors::Error; -use googapis::google::api::MonitoredResource; -use googapis::google::logging::r#type::LogSeverity; +use google_api_proto::google::api::MonitoredResource; +use google_api_proto::google::logging::r#type::LogSeverity; use serde::Deserialize; use simd_json::OwnedValue; -use std::collections::HashMap; -use tonic::transport::Channel; +use std::collections::BTreeMap; use tremor_pipeline::ConfigImpl; #[derive(Deserialize, Clone)] @@ -102,7 +100,7 @@ pub(crate) struct Config { /// This setting sets a default set of labels that can be overriden on a per event /// basis through metadata #[serde(default = "Default::default")] - pub labels: HashMap, + pub labels: BTreeMap, /// This settings sets an upper limit on the number of concurrent in flight requests /// that can be in progress simultaneously @@ -169,8 +167,8 @@ impl Config { Ok(self.default_severity) } - pub(crate) fn labels(meta: Option<&Value>) -> HashMap { - let mut labels = HashMap::new(); + pub(crate) fn labels(meta: Option<&Value>) -> BTreeMap { + let mut labels = BTreeMap::new(); if let Some(has_meta) = meta.get_object("labels") { for (k, v) in has_meta.iter() { @@ -197,8 +195,8 @@ fn value_to_monitored_resource( let kind = from.get("type"); let kind = kind.as_str(); let maybe_labels = from.get("labels"); - let labels: HashMap = match maybe_labels { - None => HashMap::new(), + let labels: BTreeMap = match maybe_labels { + None => BTreeMap::new(), Some(labels) => labels .as_object() .ok_or_else(|| { @@ -243,11 +241,7 @@ impl Connector for Gcl { sink_context: SinkContext, builder: SinkManagerBuilder, ) -> Result> { - let sink = GclSink::::new( - self.config.clone(), - builder.reply_tx(), - TonicChannelFactory, - ); + let sink = GclSink::new(self.config.clone(), builder.reply_tx()); builder.spawn(sink, sink_context).map(Some) } diff --git a/src/connectors/impls/gcl/writer/meta.rs b/src/connectors/impls/gcl/writer/meta.rs index 9e90e35435..b48feeaa4c 100644 --- a/src/connectors/impls/gcl/writer/meta.rs +++ b/src/connectors/impls/gcl/writer/meta.rs @@ -13,9 +13,9 @@ // limitations under the License. use crate::errors::Result; -use googapis::google::logging::{ +use google_api_proto::google::logging::{ r#type::HttpRequest, - v2::{LogEntryOperation, LogEntrySourceLocation}, + v2::{LogEntryOperation, LogEntrySourceLocation, LogSplit}, }; use tremor_value::Value; use value_trait::ValueAccess; @@ -28,6 +28,7 @@ pub(crate) fn insert_id(meta: Option<&Value>) -> String { get_or_default(meta, "insert_id") } +#[allow(clippy::cast_possible_wrap)] // casting seconds to u64 here is safe pub(crate) fn http_request(meta: Option<&Value>) -> Option { // Override for a specific per event trace let meta = meta?; @@ -69,7 +70,14 @@ pub(crate) fn http_request(meta: Option<&Value>) -> Option { .to_string(), latency: match http_request.get("latency").as_u64().unwrap_or(0) { 0 => None, - otherwise => Some(std::time::Duration::from_nanos(otherwise).into()), + otherwise => { + let mut duration = prost_types::Duration { + seconds: otherwise as i64 / 1_000_000_000i64, + nanos: (otherwise % 1_000_000_000) as i32, + }; + duration.normalize(); + Some(duration) + } }, cache_lookup: http_request.get("cache_lookup").as_bool().unwrap_or(false), cache_hit: http_request.get("cache_hit").as_bool().unwrap_or(false), @@ -138,14 +146,29 @@ pub(crate) fn source_location(meta: Option<&Value>) -> Option) -> Option { + let has_meta = meta?; + + if let Some(split) = has_meta.get("split") { + return Some(LogSplit { + uid: split.get("uid").as_str().unwrap_or("").to_string(), + index: split.get("line").as_i32().unwrap_or(0), + total_splits: split.get("total_splits").as_i32().unwrap_or(0), + }); + } + + // Otherwise, None as mapping is optional + None +} + #[cfg(test)] mod test { use super::super::Config; use super::*; use crate::connectors::impls::gcl::writer::default_log_severity; - use googapis::google::logging::r#type::LogSeverity; - use std::collections::HashMap as StdHashMap; + use google_api_proto::google::logging::r#type::LogSeverity; + use std::collections::BTreeMap; use tremor_pipeline::ConfigImpl; use tremor_value::literal; use tremor_value::structurize; @@ -161,7 +184,7 @@ mod test { assert_eq!(1_000_000_000, config.connect_timeout); assert_eq!(10_000_000_000, config.request_timeout); assert_eq!(LogSeverity::Default as i32, config.default_severity); - assert_eq!(std::collections::HashMap::new(), config.labels); + assert_eq!(std::collections::BTreeMap::new(), config.labels); Ok(()) } @@ -177,7 +200,7 @@ mod test { ); assert_eq!(String::new(), insert_id(Some(&meta))); assert_eq!(None, http_request(Some(&meta))); - assert_eq!(StdHashMap::new(), Config::labels(Some(&meta))); + assert_eq!(BTreeMap::new(), Config::labels(Some(&meta))); assert_eq!(None, operation(Some(&meta))); assert_eq!(String::new(), trace(Some(&meta))); assert_eq!(String::new(), span_id(Some(&meta))); @@ -319,7 +342,7 @@ mod test { // Common labels are sent once per batch of events // Metadata override ( per event ) labels are per event // So, although odd, this test is as intended - assert_eq!(StdHashMap::new(), Config::labels(None)); + assert_eq!(BTreeMap::new(), Config::labels(None)); let ok_config = Config::new(&literal!({ "labels": { "snot": "badger" } }))?; assert_eq!(1, ok_config.labels.len()); @@ -422,4 +445,24 @@ mod test { sl ); } + + #[test] + fn log_splits() { + let meta = literal!({ + "split": { + "uid": "snot", + "index": 0, + "total_splits": 5, + } + }); + let split = split(Some(&meta)); + assert_eq!( + Some(LogSplit { + uid: "snot".to_string(), + index: 0, + total_splits: 5 + }), + split + ); + } } diff --git a/src/connectors/impls/gcl/writer/sink.rs b/src/connectors/impls/gcl/writer/sink.rs index f2b93a98f1..e6233f0b51 100644 --- a/src/connectors/impls/gcl/writer/sink.rs +++ b/src/connectors/impls/gcl/writer/sink.rs @@ -14,53 +14,46 @@ use super::meta; -use crate::connectors::google::{AuthInterceptor, ChannelFactory, TokenProvider}; +use crate::connectors::google::{MockServiceRpcCall, TremorGoogleAuthz}; use crate::connectors::impls::gcl::writer::Config; use crate::connectors::prelude::*; use crate::connectors::sink::concurrency_cap::ConcurrencyCap; use crate::connectors::utils::pb; use async_std::channel::Sender; use async_std::prelude::FutureExt; -use googapis::google::logging::v2::log_entry::Payload; -use googapis::google::logging::v2::logging_service_v2_client::LoggingServiceV2Client; -use googapis::google::logging::v2::{LogEntry, WriteLogEntriesRequest}; +use google_api_proto::google::logging::v2::log_entry::Payload; +use google_api_proto::google::logging::v2::logging_service_v2_client::LoggingServiceV2Client; +use google_api_proto::google::logging::v2::{LogEntry, WriteLogEntriesRequest}; use prost_types::Timestamp; use std::time::Duration; -use tonic::codegen::InterceptedService; -use tonic::transport::{Certificate, Channel, ClientTlsConfig}; +use tonic::transport::{Channel, ClientTlsConfig}; use tonic::Code; use tremor_common::time::nanotime; +// use tremor_common::ids::SinkId; -pub(crate) struct TonicChannelFactory; +// +// NOTE This code now depends on a different protocol buffers and tonic library +// that in turn introduce a different GCP auth mechanism +// -#[async_trait::async_trait] -impl ChannelFactory for TonicChannelFactory { - async fn make_channel(&self, connect_timeout: Duration) -> Result { - let tls_config = ClientTlsConfig::new() - .ca_certificate(Certificate::from_pem(googapis::CERTIFICATES)) - .domain_name("logging.googleapis.com"); - - Ok(Channel::from_static("https://logging.googleapis.com") - .connect_timeout(connect_timeout) - .tls_config(tls_config)? - .connect() - .await?) - } +async fn make_tonic_channel(connect_timeout: Duration) -> Result { + let tls_config = ClientTlsConfig::new().domain_name("logging.googleapis.com"); + + let raw_channel = Channel::from_static("https://logging.googleapis.com") + .connect_timeout(connect_timeout) + .tls_config(tls_config)? + .connect() + .await?; + + TremorGoogleAuthz::new(raw_channel).await } -pub(crate) struct GclSink -where - T: TokenProvider + Clone, - TChannel: tonic::codegen::Service< - http::Request, - Response = http::Response, - > + Clone, -{ - client: Option>>>, +pub(crate) struct GclSink { + client: Option>, config: Config, concurrency_cap: ConcurrencyCap, reply_tx: Sender, - channel_factory: Box + Send + Sync>, + mock_logic: Option, } fn value_to_log_entry( @@ -84,24 +77,28 @@ fn value_to_log_entry( trace_sampled: meta::trace_sampled(meta)?, source_location: meta::source_location(meta), payload: Some(Payload::JsonPayload(pb::value_to_prost_struct(data)?)), + split: meta::split(meta), }) } -impl< - T: TokenProvider + Clone, - TChannel: tonic::codegen::Service< - http::Request, - Response = http::Response, - Error = TChannelError, - > + Send - + Clone, - TChannelError: Into>, - > GclSink -{ - pub fn new( +impl GclSink { + pub fn new(config: Config, reply_tx: Sender) -> Self { + let concurrency_cap = ConcurrencyCap::new(config.concurrency, reply_tx.clone()); + Self { + client: None, + config, + concurrency_cap, + reply_tx, + mock_logic: None, + } + } + + #[cfg(test)] + #[cfg(feature = "gcp-integration")] + pub fn new_mock( config: Config, reply_tx: Sender, - channel_factory: impl ChannelFactory + Send + Sync + 'static, + mock_logic: MockServiceRpcCall, ) -> Self { let concurrency_cap = ConcurrencyCap::new(config.concurrency, reply_tx.clone()); Self { @@ -109,26 +106,13 @@ impl< config, concurrency_cap, reply_tx, - channel_factory: Box::new(channel_factory), + mock_logic: Some(mock_logic), } } } #[async_trait::async_trait] -impl< - T: TokenProvider + Clone + 'static, - TChannel: tonic::codegen::Service< - http::Request, - Response = http::Response, - Error = TChannelError, - > + Send - + Clone - + 'static, - TChannelError: Into> + Send + Sync, - > Sink for GclSink -where - TChannel::Future: Send, -{ +impl Sink for GclSink { async fn on_event( &mut self, _input: &str, @@ -180,7 +164,7 @@ where .await?; if let Err(error) = log_entries_response { - error!("Failed to write a log entries: {}", error); + error!("Failed to write log entries: {}", error.message()); if matches!( error.code(), @@ -219,20 +203,20 @@ where } async fn connect(&mut self, ctx: &SinkContext, _attempt: &Attempt) -> Result { - info!("{} Connecting to Google Cloud Logging", ctx); - let channel = self - .channel_factory - .make_channel(Duration::from_nanos(self.config.connect_timeout)) - .await?; + if let Some(logic) = self.mock_logic { + info!("{} Mocking connection to Google Cloud Logging", ctx); + self.client = Some(LoggingServiceV2Client::new(TremorGoogleAuthz::new_mock( + logic, + ))); + } else { + info!("{} Connecting to Google Cloud Logging", ctx); + let channel = + make_tonic_channel(Duration::from_nanos(self.config.connect_timeout)).await?; - let client = LoggingServiceV2Client::with_interceptor( - channel, - AuthInterceptor { - token_provider: T::default(), - }, - ); + let client = LoggingServiceV2Client::new(channel); - self.client = Some(client); + self.client = Some(client); + } Ok(true) } @@ -253,68 +237,38 @@ mod test { use super::*; use crate::connectors::impls::gcl; use crate::connectors::tests::ConnectorHarness; + use crate::connectors::utils::quiescence::QuiescenceBeacon; use crate::connectors::ConnectionLostNotifier; - use crate::connectors::{ - google::tests::TestTokenProvider, utils::quiescence::QuiescenceBeacon, - }; use async_std::channel::bounded; - use bytes::Bytes; - use futures::future::Ready; - use googapis::google::logging::r#type::LogSeverity; - use googapis::google::logging::v2::WriteLogEntriesResponse; + use futures::executor::block_on; + use google_api_proto::google::logging::{r#type::LogSeverity, v2::WriteLogEntriesResponse}; use http::{HeaderMap, HeaderValue}; - use http_body::Body; - use prost::Message; - use std::task::Poll; - use std::{ - collections::HashMap, - fmt::{Debug, Display, Formatter}, - }; - use tonic::body::BoxBody; - use tonic::codegen::Service; use tremor_common::ids::SinkId; use tremor_pipeline::CbAction::Trigger; use tremor_pipeline::EventId; use tremor_value::{literal, structurize}; - #[derive(Debug)] - enum MockServiceError {} - - impl Display for MockServiceError { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "MockServiceError") - } - } - - impl std::error::Error for MockServiceError {} - - struct MockChannelFactory; - - #[async_trait::async_trait] - impl ChannelFactory for MockChannelFactory { - async fn make_channel(&self, _connect_timeout: Duration) -> Result { - Ok(MockService {}) - } - } - - #[derive(Clone)] - struct MockService {} - - impl Service> for MockService { - type Response = http::Response; - type Error = MockServiceError; - type Future = - Ready, MockServiceError>>; - - fn poll_ready( - &mut self, - _cx: &mut std::task::Context<'_>, - ) -> Poll> { - Poll::Ready(Ok(())) - } - - #[allow(clippy::unwrap_used, clippy::cast_possible_truncation)] // We don't control the return type here - fn call(&mut self, _request: http::Request) -> Self::Future { + #[async_std::test] + async fn on_event_can_send_an_event() -> Result<()> { + let (tx, rx) = bounded(10); + let (connection_lost_tx, _connection_lost_rx) = bounded(10); + let config = Config { + log_name: None, + resource: None, + partial_success: false, + dry_run: false, + connect_timeout: 0, + request_timeout: 0, + default_severity: 0, + labels: Default::default(), + concurrency: 0, + }; + let mut sink = GclSink::new_mock(config, tx, |_req| { + // TODO As mock logic implementations become common, convenience functions + // should be refactored/extracted as appropriate. + use bytes::Bytes; + use http_body::Body; + use prost::Message; let mut buffer = vec![]; WriteLogEntriesResponse {} @@ -324,7 +278,7 @@ mod test { let body = http_body::Full::new(body); let body = http_body::combinators::BoxBody::new(body).map_err(|err| match err {}); let mut response = tonic::body::BoxBody::new(body); - let (mut tx, body) = tonic::transport::Body::channel(); + let (mut tx, body2) = tonic::transport::Body::channel(); let jh = async_std::task::spawn(async move { let response = response.data().await.unwrap().unwrap(); let len: [u8; 4] = (response.len() as u32).to_ne_bytes(); @@ -340,33 +294,12 @@ mod test { tx.send_trailers(trailers).await.unwrap(); }); async_std::task::spawn_blocking(|| jh); - - let response = http::Response::new(body); - - futures::future::ready(Ok(response)) - } - } - - #[async_std::test] - async fn on_event_can_send_an_event() -> Result<()> { - let (tx, rx) = bounded(10); - let (connection_lost_tx, _connection_lost_rx) = bounded(10); - - let mut sink = GclSink::::new( - Config { - log_name: None, - resource: None, - partial_success: false, - dry_run: false, - connect_timeout: 0, - request_timeout: 0, - default_severity: 0, - labels: HashMap::default(), - concurrency: 0, - }, - tx, - MockChannelFactory, - ); + let body: Bytes = block_on(async { hyper::body::to_bytes(body2).await.unwrap() }); + let body = http_body::Full::new(body); + let body = http_body::combinators::BoxBody::new(body).map_err(|err| match err {}); + let response = tonic::body::BoxBody::new(body); + http::Response::new(response) + }); let sink_context = SinkContext { uid: SinkId::default(), alias: Alias::new("a", "b"), @@ -454,7 +387,7 @@ mod test { "connect_timeout": 1_000_000 }))?; - let mut sink = GclSink::::new(config, reply_tx, MockChannelFactory); + let mut sink = GclSink::new(config, reply_tx); let result = sink .on_event( diff --git a/src/connectors/impls/gpubsub/consumer.rs b/src/connectors/impls/gpubsub/consumer.rs index 3a877e9484..c0305c38cd 100644 --- a/src/connectors/impls/gpubsub/consumer.rs +++ b/src/connectors/impls/gpubsub/consumer.rs @@ -12,32 +12,34 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::connectors::google::{AuthInterceptor, TokenProvider}; - +use crate::connectors::google::TremorGoogleAuthz; use crate::connectors::prelude::*; use crate::connectors::utils::url::HttpsDefaults; use async_std::channel::{Receiver, Sender}; use async_std::stream::StreamExt; use async_std::sync::RwLock; -use async_std::task; +// use async_std::task; use async_std::task::JoinHandle; use beef::generic::Cow; -use googapis::google::pubsub::v1::subscriber_client::SubscriberClient; -use googapis::google::pubsub::v1::{ +use google_api_proto::google::pubsub::v1::subscriber_client::SubscriberClient; +use google_api_proto::google::pubsub::v1::{ GetSubscriptionRequest, PubsubMessage, ReceivedMessage, StreamingPullRequest, }; use serde::Deserialize; -use std::collections::HashMap; +use std::collections::BTreeMap; use std::fmt::Debug; -use std::marker::PhantomData; use std::sync::Arc; use std::time::{Duration, SystemTime}; -use tonic::codegen::InterceptedService; -use tonic::transport::{Certificate, Channel, ClientTlsConfig}; -use tonic::{Code, Status}; +use tonic::transport::{Channel, ClientTlsConfig}; +use tonic::Code; use tremor_common::blue_green_hashmap::BlueGreenHashMap; use tremor_pipeline::ConfigImpl; +// +// NOTE This code now depends on a different protocol buffers and tonic library +// that in turn introduce a different GCP auth mechanism +// + // controlling retries upon gpubsub returning `Unavailable` from StreamingPull // this in on purpose not exposed via config as this should remain an internal thing const RETRY_WAIT_INTERVAL: Duration = Duration::from_secs(1); @@ -63,12 +65,6 @@ fn default_ack_deadline() -> u64 { #[derive(Debug, Default)] pub(crate) struct Builder {} -#[cfg(all(test, feature = "gcp-integration"))] -type GSubWithTokenProvider = GSub; - -#[cfg(not(all(test, feature = "gcp-integration")))] -type GSubWithTokenProvider = GSub; - #[async_trait::async_trait] impl ConnectorBuilder for Builder { fn connector_type(&self) -> ConnectorType { @@ -77,37 +73,36 @@ impl ConnectorBuilder for Builder { async fn build_cfg( &self, - alias: &Alias, + _alias: &Alias, _: &ConnectorConfig, raw: &Value, _kill_switch: &KillSwitch, ) -> Result> { let config = Config::new(raw)?; let url = Url::::parse(config.url.as_str())?; - let client_id = format!("tremor-{}-{alias}-{:?}", hostname(), task::current().id()); - - Ok(Box::new(GSubWithTokenProvider { + // let client_id = format!("tremor-{}-{alias}-{:?}", hostname(), task::current().id()); + let client_id = "snot".to_string(); + Ok(Box::new(GSub { config, url, client_id, - _phantom: PhantomData::default(), })) } } -struct GSub { +struct GSub { config: Config, url: Url, client_id: String, - _phantom: PhantomData, } -type PubSubClient = SubscriberClient>>; +type PubSubClient = SubscriberClient; type AsyncTaskMessage = Result<(u64, PubsubMessage)>; -struct GSubSource { +struct GSubSource { config: Config, - client: Option>, + hostname: String, + // client: Option, receiver: Option>, ack_sender: Option>, task_handle: Option>, @@ -115,22 +110,33 @@ struct GSubSource { client_id: String, } -impl GSubSource { - pub fn new(config: Config, url: Url, client_id: String) -> Self { - GSubSource { +impl GSubSource { + pub fn new(config: Config, url: Url, client_id: String) -> Result { + let hostname = config + .url + .host_str() + .ok_or_else(|| { + ErrorKind::InvalidConfiguration( + "gpubsub-consumer".to_string(), + "Missing hostname".to_string(), + ) + })? + .to_string(); + Ok(GSubSource { config, + hostname, url, client_id, - client: None, + // client: None, receiver: None, task_handle: None, ack_sender: None, - } + }) } } -async fn consumer_task( - mut client: PubSubClient, +async fn consumer_task( + mut client: PubSubClient, ctx: SourceContext, client_id: String, sender: Sender, @@ -244,7 +250,7 @@ fn pubsub_metadata( id: String, ordering_key: String, publish_time: Option, - attributes: HashMap, + attributes: BTreeMap, ) -> Value<'static> { let mut attributes_value = Value::object_with_capacity(attributes.len()); for (name, value) in attributes { @@ -263,19 +269,14 @@ fn pubsub_metadata( } #[async_trait::async_trait] -impl Source for GSubSource { +impl Source for GSubSource { async fn connect(&mut self, ctx: &SourceContext, _attempt: &Attempt) -> Result { let mut channel = Channel::from_shared(self.config.url.to_string())? .connect_timeout(Duration::from_nanos(self.config.connect_timeout)); if self.url.scheme() == "https" { let tls_config = ClientTlsConfig::new() - .ca_certificate(Certificate::from_pem(googapis::CERTIFICATES)) - .domain_name( - self.url - .host_str() - .ok_or_else(|| Status::unavailable("The endpoint is missing a hostname"))? - .to_string(), - ); + // TODO FIXME .ca_certificate(Certificate::from_pem(googapis::CERTIFICATES)) + .domain_name(self.hostname.clone()); channel = channel.tls_config(tls_config)?; } @@ -286,12 +287,7 @@ impl Source for GSubSource { task_handle.cancel().await; } - let mut client = SubscriberClient::with_interceptor( - channel.clone(), - AuthInterceptor { - token_provider: T::default(), - }, - ); + let mut client = SubscriberClient::new(TremorGoogleAuthz::new(channel.clone()).await?); // check that the subscription exists let res = client .get_subscription(GetSubscriptionRequest { @@ -305,7 +301,10 @@ impl Source for GSubSource { ); debug!("{ctx} Subscription details {res:?}"); - let client_background = client.clone(); + // let client_background = client.clone(); + // let client_background = SubscriberClient::new(auth_channel.clone()); + let client_background = + SubscriberClient::new(TremorGoogleAuthz::new(channel.clone()).await?); let (tx, rx) = async_std::channel::bounded(QSIZE.load(Ordering::Relaxed)); let (ack_tx, ack_rx) = async_std::channel::bounded(QSIZE.load(Ordering::Relaxed)); @@ -325,7 +324,7 @@ impl Source for GSubSource { self.receiver = Some(rx); self.ack_sender = Some(ack_tx); - self.client = Some(client); + // self.client = Some(client); self.task_handle = Some(join_handle); Ok(true) @@ -345,7 +344,7 @@ impl Source for GSubSource { Ok(SourceReply::Data { origin_uri: EventOriginUri::default(), - data: pubsub_message.data, + data: pubsub_message.data.into(), meta: Some(pubsub_metadata( pubsub_message.message_id, pubsub_message.ordering_key, @@ -386,17 +385,17 @@ impl Source for GSubSource { } #[async_trait::async_trait] -impl Connector for GSub { +impl Connector for GSub { async fn create_source( &mut self, source_context: SourceContext, builder: SourceManagerBuilder, ) -> Result> { - let source = GSubSource::::new( + let source = GSubSource::new( self.config.clone(), self.url.clone(), self.client_id.clone(), - ); + )?; builder.spawn(source, source_context).map(Some) } diff --git a/src/connectors/impls/gpubsub/producer.rs b/src/connectors/impls/gpubsub/producer.rs index 53514489ca..318617ce8a 100644 --- a/src/connectors/impls/gpubsub/producer.rs +++ b/src/connectors/impls/gpubsub/producer.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::connectors::google::{AuthInterceptor, TokenProvider}; +use crate::connectors::google::TremorGoogleAuthz; use crate::connectors::prelude::{ Alias, Attempt, ErrorKind, EventSerializer, KillSwitch, SinkAddr, SinkContext, SinkManagerBuilder, SinkReply, Url, @@ -25,13 +25,11 @@ use crate::connectors::{ }; use crate::errors::Result; use async_std::prelude::FutureExt; -use googapis::google::pubsub::v1::publisher_client::PublisherClient; -use googapis::google::pubsub::v1::{PublishRequest, PubsubMessage}; -use std::collections::HashMap; -use std::marker::PhantomData; +use google_api_proto::google::pubsub::v1::publisher_client::PublisherClient; +use google_api_proto::google::pubsub::v1::{PublishRequest, PubsubMessage}; +use std::collections::BTreeMap; use std::time::Duration; -use tonic::codegen::InterceptedService; -use tonic::transport::{Certificate, Channel, ClientTlsConfig}; +use tonic::transport::{Channel, ClientTlsConfig}; use tonic::Code; use tremor_pipeline::{ConfigImpl, Event}; use tremor_value::Value; @@ -54,13 +52,6 @@ impl ConfigImpl for Config {} #[derive(Default, Debug)] pub(crate) struct Builder {} -#[cfg(all(test, feature = "gcp-integration"))] -type GpubConnectorWithTokenProvider = - GpubConnector; - -#[cfg(not(all(test, feature = "gcp-integration")))] -type GpubConnectorWithTokenProvider = GpubConnector; - #[async_trait::async_trait()] impl ConnectorBuilder for Builder { fn connector_type(&self) -> ConnectorType { @@ -76,26 +67,22 @@ impl ConnectorBuilder for Builder { ) -> Result> { let config = Config::new(raw_config)?; - Ok(Box::new(GpubConnectorWithTokenProvider { - config, - _phantom: PhantomData::default(), - })) + Ok(Box::new(GpubConnector { config })) } } -struct GpubConnector { +struct GpubConnector { config: Config, - _phantom: PhantomData, } #[async_trait::async_trait()] -impl Connector for GpubConnector { +impl Connector for GpubConnector { async fn create_sink( &mut self, sink_context: SinkContext, builder: SinkManagerBuilder, ) -> Result> { - let sink = GpubSink:: { + let sink = GpubSink { config: self.config.clone(), hostname: self .config @@ -122,21 +109,21 @@ impl Connector for GpubConnector { } } -struct GpubSink { +struct GpubSink { config: Config, hostname: String, - client: Option>>>, + client: Option>, } #[async_trait::async_trait()] -impl Sink for GpubSink { +impl Sink for GpubSink { async fn connect(&mut self, _ctx: &SinkContext, _attempt: &Attempt) -> Result { let mut channel = Channel::from_shared(self.config.url.to_string())? .connect_timeout(Duration::from_nanos(self.config.connect_timeout)); if self.config.url.scheme() == "https" { let tls_config = ClientTlsConfig::new() - .ca_certificate(Certificate::from_pem(googapis::CERTIFICATES)) + // TODO FIXME .ca_certificate(Certificate::from_pem(googapis::CERTIFICATES)) .domain_name(self.hostname.clone()); channel = channel.tls_config(tls_config)?; @@ -144,12 +131,7 @@ impl Sink for GpubSink { let channel = channel.connect().await?; - self.client = Some(PublisherClient::with_interceptor( - channel, - AuthInterceptor { - token_provider: T::default(), - }, - )); + self.client = Some(PublisherClient::new(TremorGoogleAuthz::new(channel).await?)); Ok(true) } @@ -178,8 +160,8 @@ impl Sink for GpubSink { .map_or_else(String::new, ToString::to_string); messages.push(PubsubMessage { - data: payload, - attributes: HashMap::new(), + data: payload.into(), + attributes: BTreeMap::new(), // publish_time and message_id will be ignored in the request and set by server message_id: String::new(), publish_time: None, @@ -240,11 +222,10 @@ impl Sink for GpubSink { #[cfg(feature = "gcp-integration")] mod tests { use super::*; - use crate::connectors::google::tests::TestTokenProvider; #[test] pub fn is_not_auto_ack() { - let sink = GpubSink:: { + let sink = GpubSink { config: Config { connect_timeout: 0, request_timeout: 0, @@ -260,7 +241,7 @@ mod tests { #[test] pub fn is_async() { - let sink = GpubSink:: { + let sink = GpubSink { config: Config { connect_timeout: 0, request_timeout: 0, diff --git a/src/connectors/impls/otel/client.rs b/src/connectors/impls/otel/client.rs index 51f16b4d37..0517c61ef4 100644 --- a/src/connectors/impls/otel/client.rs +++ b/src/connectors/impls/otel/client.rs @@ -12,20 +12,25 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::{common::OtelDefaults, logs, metrics, trace}; +use super::{ + common::{Compression, OtelDefaults}, + logs, metrics, trace, +}; use crate::connectors::prelude::*; -use tonic::transport::Channel as TonicChannel; use tonic::transport::Endpoint as TonicEndpoint; -use tremor_otelapis::opentelemetry::proto::collector::{ - logs::v1::{logs_service_client::LogsServiceClient, ExportLogsServiceRequest}, - metrics::v1::{metrics_service_client::MetricsServiceClient, ExportMetricsServiceRequest}, - trace::v1::{trace_service_client::TraceServiceClient, ExportTraceServiceRequest}, +use tonic::{codegen::CompressionEncoding, transport::Channel as TonicChannel}; +use tremor_otelapis::{ + common::FallibleOtelResponse, + opentelemetry::proto::collector::{ + logs::v1::{logs_service_client::LogsServiceClient, ExportLogsServiceRequest}, + metrics::v1::{metrics_service_client::MetricsServiceClient, ExportMetricsServiceRequest}, + trace::v1::{trace_service_client::TraceServiceClient, ExportTraceServiceRequest}, + }, }; const CONNECTOR_TYPE: &str = "otel_client"; // TODO Consider concurrency cap? - #[derive(Debug, Clone, Deserialize, Default)] #[serde(deny_unknown_fields)] pub(crate) struct Config { @@ -40,6 +45,9 @@ pub(crate) struct Config { /// Enables the metrics service #[serde(default = "default_true")] pub(crate) metrics: bool, + /// Configurable compression for otel payloads + #[serde(default = "Default::default")] + pub(crate) compression: Compression, } impl ConfigImpl for Config {} @@ -47,7 +55,6 @@ impl ConfigImpl for Config {} /// The `OpenTelemetry` client connector pub(crate) struct Client { config: Config, - origin_uri: EventOriginUri, } // #[cfg_attr(coverage, no_coverage)] @@ -74,14 +81,7 @@ impl ConnectorBuilder for Builder { _kill_switch: &KillSwitch, ) -> Result> { let config = Config::new(config)?; - let origin_uri = EventOriginUri { - scheme: "tremor-otel-client".to_string(), - host: config.url.host_or_local().to_string(), - port: config.url.port(), - path: vec![], - }; - - Ok(Box::new(Client { config, origin_uri })) + Ok(Box::new(Client { config })) } } @@ -104,7 +104,6 @@ impl Connector for Client { builder: SinkManagerBuilder, ) -> Result> { let sink = OtelSink { - origin_uri: self.origin_uri.clone(), config: self.config.clone(), remote: None, }; @@ -112,13 +111,30 @@ impl Connector for Client { } } -#[allow(dead_code)] struct OtelSink { - origin_uri: EventOriginUri, config: Config, remote: Option, } +fn handle_fallible_otel_response( + response: std::result::Result, tonic::Status>, +) -> Option +where + FallibleOtelResponse: std::convert::From, +{ + match response { + Ok(response) => { + let partial: FallibleOtelResponse = response.into_inner().into(); + if partial.is_ok() { + None + } else { + Some(partial.error_message.into()) // FIXME make this a nice hygienic error ( omits counts etc... ) + } + } + Err(e) => Some(e.into()), + } +} + #[async_trait::async_trait()] impl Sink for OtelSink { async fn connect(&mut self, _ctx: &SinkContext, _attempt: &Attempt) -> Result { @@ -128,10 +144,29 @@ impl Sink for OtelSink { .connect() .await?; + let logs_client = LogsServiceClient::new(channel.clone()); + let metrics_client = MetricsServiceClient::new(channel.clone()); + let trace_client = TraceServiceClient::new(channel); + + let (logs_client, metrics_client, trace_client) = match self.config.compression { + Compression::Gzip => ( + logs_client + .send_compressed(CompressionEncoding::Gzip) + .accept_compressed(CompressionEncoding::Gzip), + metrics_client + .send_compressed(CompressionEncoding::Gzip) + .accept_compressed(CompressionEncoding::Gzip), + trace_client + .send_compressed(CompressionEncoding::Gzip) + .accept_compressed(CompressionEncoding::Gzip), + ), + Compression::None => (logs_client, metrics_client, trace_client), + }; + self.remote = Some(RemoteOpenTelemetryEndpoint { - logs_client: LogsServiceClient::new(channel.clone()), - metrics_client: MetricsServiceClient::new(channel.clone()), - trace_client: TraceServiceClient::new(channel), + logs_client, + metrics_client, + trace_client, }); Ok(true) @@ -146,14 +181,14 @@ impl Sink for OtelSink { ) -> Result { if let Some(remote) = &mut self.remote { for value in event.value_iter() { - let err = if self.config.metrics && value.contains_key("metrics") { + let err: Option = if self.config.metrics && value.contains_key("metrics") { let request = ExportMetricsServiceRequest { resource_metrics: ctx.bail_err( metrics::resource_metrics_to_pb(Some(value)), "Error converting payload to otel metrics", )?, }; - remote.metrics_client.export(request).await.err() + handle_fallible_otel_response(remote.metrics_client.export(request).await) } else if self.config.logs && value.contains_key("logs") { let request = ExportLogsServiceRequest { resource_logs: ctx.bail_err( @@ -161,15 +196,15 @@ impl Sink for OtelSink { "Error converting payload to otel logs", )?, }; - remote.logs_client.export(request).await.err() + handle_fallible_otel_response(remote.logs_client.export(request).await) } else if self.config.trace && value.contains_key("trace") { let request = ExportTraceServiceRequest { resource_spans: ctx.bail_err( - trace::resource_spans_to_pb(Some(value)), + trace::resource_spans_to_pb(value), "Error converting payload to otel span", )?, }; - remote.trace_client.export(request).await.err() + handle_fallible_otel_response(remote.trace_client.export(request).await) } else { warn!("{ctx} Invalid or disabled otel payload: {value}"); None diff --git a/src/connectors/impls/otel/common.rs b/src/connectors/impls/otel/common.rs index d93262eb47..d11d84e885 100644 --- a/src/connectors/impls/otel/common.rs +++ b/src/connectors/impls/otel/common.rs @@ -16,9 +16,11 @@ use crate::connectors::utils::{pb, url}; use crate::errors::Result; +use beef::Cow; +use halfbrown::HashMap; use simd_json::Builder; use tremor_otelapis::opentelemetry::proto::common::v1::{ - any_value, AnyValue, ArrayValue, InstrumentationLibrary, KeyValue, KeyValueList, StringKeyValue, + any_value, AnyValue, ArrayValue, InstrumentationScope, KeyValue, KeyValueList, }; use tremor_value::{literal, StaticNode, Value}; use value_trait::ValueAccess; @@ -32,6 +34,15 @@ impl url::Defaults for OtelDefaults { const PORT: u16 = 4317; } +/// Enum to support configurable compression on otel grpc client/servers. +#[derive(Clone, Copy, Debug, Default, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub(crate) enum Compression { + #[default] + None, + Gzip, +} + pub(crate) const EMPTY: Vec = Vec::new(); pub(crate) fn any_value_to_json(pb: AnyValue) -> Value<'static> { @@ -43,7 +54,6 @@ pub(crate) fn any_value_to_json(pb: AnyValue) -> Value<'static> { Some(Inner::DoubleValue(v)) => v.into(), Some(Inner::ArrayValue(v)) => v.values.into_iter().map(any_value_to_json).collect(), Some(Inner::KvlistValue(v)) => { - // let mut record = HashMap::with_capacity(v.values.len()); v.values .into_iter() .map(|e| { @@ -53,8 +63,6 @@ pub(crate) fn any_value_to_json(pb: AnyValue) -> Value<'static> { ) }) .collect() - - // Value::from(record) } Some(Inner::BytesValue(b)) => Value::Bytes(b.into()), None => Value::null(), @@ -110,32 +118,91 @@ pub(crate) fn any_value_to_pb(data: &Value<'_>) -> AnyValue { } } +pub(crate) fn maybe_instrumentation_scope_to_json( + pb: Option, +) -> Option> { + if let Some(pb) = pb { + return Some(literal!({ + "name": pb.name.clone(), + "version": pb.version.clone(), + "attributes": key_value_list_to_json(pb.attributes), + "dropped_attributes_count": 0 + })); + } + + None +} + +pub(crate) fn maybe_instrumentation_scope_to_pb( + data: Option<&Value<'_>>, +) -> Result> { + if let Some(data) = data { + let name = data + .get("name") + .and_then(Value::as_str) + .unwrap_or_default() + .to_string(); + let version = data + .get("version") + .and_then(Value::as_str) + .unwrap_or_default() + .to_string(); + let attributes = maybe_key_value_list_to_pb(data.get("attributes"))?; + let dropped_attributes_count = + pb::maybe_int_to_pbu32(data.get("dropped_attributes_count")).unwrap_or_default(); + Ok(Some(InstrumentationScope { + name, + version, + attributes, + dropped_attributes_count, + })) + } else { + Ok(None) + } +} + pub(crate) fn maybe_any_value_to_json(pb: Option) -> Option> { pb.map(any_value_to_json) } -pub(crate) fn string_key_value_to_json(pb: Vec) -> Value<'static> { +pub(crate) fn string_key_value_to_json(pb: Vec) -> Value<'static> { + // REFACTOR whack as string_key_value was deprecated-removed pb.into_iter() - .map(|StringKeyValue { key, value }| (key, value)) + .map(|KeyValue { key, value }| (key, maybe_any_value_to_json(value))) .collect() } -pub(crate) fn string_key_value_to_pb(data: Option<&Value<'_>>) -> Result> { +pub(crate) fn string_key_value_to_pb(data: Option<&Value<'_>>) -> Result> { + // REFACTOR whack as string_key_value was deprecated-removed data.as_object() .ok_or("Unable to map json to Vec pb")? .iter() .map(|(key, value)| { let key = key.to_string(); - let value = pb::maybe_string_to_pb(Some(value))?; - Ok(StringKeyValue { key, value }) + let value = any_value_to_pb(value); + Ok(KeyValue { + key, + value: Some(value), + }) }) .collect() } pub(crate) fn key_value_list_to_json(pb: Vec) -> Value<'static> { - pb.into_iter() - .map(|KeyValue { key, value }| (key, maybe_any_value_to_json(value))) - .collect() + let kv: HashMap, Value<'_>> = pb + .into_iter() + .filter_map(|e| match e { + KeyValue { + key, + value: Some(value), + } => { + let k = Cow::owned(key); + Some((k, any_value_to_json(value))) + } + KeyValue { .. } => None, + }) + .collect(); + Value::Object(Box::new(kv)) } pub(crate) fn maybe_key_value_list_to_pb(data: Option<&Value<'_>>) -> Result> { @@ -145,7 +212,7 @@ pub(crate) fn maybe_key_value_list_to_pb(data: Option<&Value<'_>>) -> Result Result> { +pub(crate) fn get_attributes(data: &Value) -> Result> { match (data.get_object("attributes"), data.get_object("labels")) { (None, None) => Err("missing field `attributes`".into()), (Some(a), None) | (None, Some(a)) => Ok(obj_key_value_list_to_pb(a)), @@ -167,25 +234,13 @@ pub(crate) fn obj_key_value_list_to_pb(data: &tremor_value::Object<'_>) -> Vec Value<'static> { - literal!({ - "name": il.name, - "version": il.version, - }) -} - -pub(crate) fn instrumentation_library_to_pb(data: &Value<'_>) -> Result { - Ok(InstrumentationLibrary { - name: pb::maybe_string_to_pb((*data).get("name"))?, - version: pb::maybe_string_to_pb((*data).get("version"))?, - }) -} - #[cfg(test)] +#[allow(clippy::unnecessary_wraps)] mod tests { #![allow(clippy::float_cmp)] use simd_json::prelude::*; use tremor_otelapis::opentelemetry::proto::common::v1::{ArrayValue, KeyValueList}; + use tremor_value::literal; use super::*; @@ -366,9 +421,9 @@ mod tests { #[test] fn string_key_value_list() -> Result<()> { - let pb = vec![StringKeyValue { + let pb = vec![KeyValue { key: "snot".into(), - value: "badger".into(), + value: Some(any_value_to_pb(&literal!("badger"))), }]; let json = string_key_value_to_json(pb.clone()); let back_again = string_key_value_to_pb(Some(&json))?; @@ -379,16 +434,24 @@ mod tests { } #[test] - fn instrumentation_library() -> Result<()> { - let pb = InstrumentationLibrary { + fn instrumentation_scope() -> Result<()> { + use any_value::Value as Inner; + let pb = InstrumentationScope { name: "name".into(), version: "v0.1.2".into(), + attributes: vec![KeyValue { + key: "snot".to_string(), + value: Some(AnyValue { + value: Some(Inner::StringValue("badger".to_string())), + }), + }], + dropped_attributes_count: 0, }; - let json = maybe_instrumentation_library_to_json(pb.clone()); - let back_again = instrumentation_library_to_pb(&json)?; - let expected: Value = literal!({"name": "name", "version": "v0.1.2"}); - assert_eq!(expected, json); - assert_eq!(pb, back_again); + let json = maybe_instrumentation_scope_to_json(Some(pb.clone())); + let back_again = maybe_instrumentation_scope_to_pb(json.as_ref())?; + let expected: Value = literal!({"name": "name", "version": "v0.1.2", "attributes": { "snot": "badger" }, "dropped_attributes_count": 0}); + assert_eq!(Some(expected), json); + assert_eq!(Some(pb), back_again); Ok(()) } } diff --git a/src/connectors/impls/otel/logs.rs b/src/connectors/impls/otel/logs.rs index 2c60b2d5f4..87695f26ff 100644 --- a/src/connectors/impls/otel/logs.rs +++ b/src/connectors/impls/otel/logs.rs @@ -15,16 +15,22 @@ #![allow(dead_code)] use super::{ - common::{self, instrumentation_library_to_pb, maybe_instrumentation_library_to_json, EMPTY}, + common::{self}, id, resource::{self, resource_to_pb}, }; -use crate::connectors::utils::pb; +use crate::connectors::{ + impls::otel::common::{key_value_list_to_json, maybe_instrumentation_scope_to_json}, + utils::pb, +}; use crate::errors::Result; +use simd_json::ValueAccess; use tremor_otelapis::opentelemetry::proto::{ collector::logs::v1::ExportLogsServiceRequest, - logs::v1::{InstrumentationLibraryLogs, LogRecord, ResourceLogs}, + common::v1::InstrumentationScope, + logs::v1::{LogRecord, ResourceLogs, ScopeLogs}, + resource::v1::Resource, }; use tremor_value::{literal, prelude::*, Value}; @@ -53,43 +59,45 @@ fn affirm_severity_number_valid(severity_number: i32) -> Result { } } -fn log_record_to_json(log: LogRecord) -> Result> { - Ok(literal!({ - "name": log.name, - "time_unix_nano": log.time_unix_nano, - "severity_number": affirm_severity_number_valid(log.severity_number)?, - "severity_text": log.severity_text.to_string(), - "flags": affirm_traceflags_valid(log.flags)?, - "span_id": id::hex_span_id_to_json(&log.span_id), - "trace_id": id::hex_trace_id_to_json(&log.trace_id), - "attributes": common::key_value_list_to_json(log.attributes), - "dropped_attributes_count": log.dropped_attributes_count, - "body": common::maybe_any_value_to_json(log.body), - })) +fn scope_log_to_json(log: ScopeLogs) -> Result> { + let mut scope_json = literal!({ + "log_records": log_records_to_json(log.log_records), + "schema_url": log.schema_url.clone() + }); + if let Some(scope) = maybe_instrumentation_scope_to_json(log.scope) { + scope_json.insert("scope", scope.clone())?; + } + + Ok(scope_json) } -pub(crate) fn instrumentation_library_logs_to_json( - pb: Vec, -) -> Result> { - pb.into_iter() - .map(|data| { - let logs = data - .logs - .into_iter() - .map(log_record_to_json) - .collect::>()?; - let mut e = literal!({ "logs": logs, "schema_url": data.schema_url }); - if let Some(il) = data.instrumentation_library { - let il = maybe_instrumentation_library_to_json(il); - e.try_insert("instrumentation_library", il); - } - Ok(e) - }) - .collect() +pub(crate) fn scope_logs_to_json(log: Vec) -> Result>> { + log.into_iter().map(scope_log_to_json).collect() +} + +pub(crate) fn log_record_to_json(pb: LogRecord) -> Value<'static> { + literal!({ + "observed_time_unix_nano": pb.observed_time_unix_nano, + "time_unix_nano": pb.time_unix_nano, + "severity_number": pb.severity_number, + "severity_text": pb.severity_text, + "body": common::maybe_any_value_to_json(pb.body), + "flags": pb.flags, + "span_id": id::hex_span_id_to_json(&pb.span_id), + "trace_id": id::hex_trace_id_to_json(&pb.trace_id), + "attributes": key_value_list_to_json(pb.attributes), + "dropped_attributes_count": pb.dropped_attributes_count, + }) +} + +pub(crate) fn log_records_to_json(log: Vec) -> Vec> { + log.into_iter().map(log_record_to_json).collect() } pub(crate) fn log_record_to_pb(log: &Value<'_>) -> Result { Ok(LogRecord { + observed_time_unix_nano: pb::maybe_int_to_pbu64(log.get("observed_time_unix_nano")) + .unwrap_or_default(), // value of 0 indicates unknown or missing timestamp time_unix_nano: pb::maybe_int_to_pbu64(log.get("time_unix_nano")).unwrap_or_default(), @@ -100,8 +108,6 @@ pub(crate) fn log_record_to_pb(log: &Value<'_>) -> Result { .unwrap_or_default(), // defined as optional - fallback to an empty string severity_text: pb::maybe_string_to_pb(log.get("severity_text")).unwrap_or_default(), - // name is defined as optional - fallback to empty string - name: pb::maybe_string_to_pb(log.get("name")).unwrap_or_default(), body: log.get("body").map(common::any_value_to_pb), flags: affirm_traceflags_valid( pb::maybe_int_to_pbu32(log.get("flags")).unwrap_or_default(), @@ -115,50 +121,16 @@ pub(crate) fn log_record_to_pb(log: &Value<'_>) -> Result { }) } -pub(crate) fn maybe_instrumentation_library_logs_to_pb( - data: Option<&Value<'_>>, -) -> Result> { - data.as_array() - .ok_or("Invalid json mapping for InstrumentationLibraryLogs")? - .iter() - .filter_map(Value::as_object) - .map(|ill| { - let logs = ill - .get("logs") - .and_then(Value::as_array) - .unwrap_or(&EMPTY) - .iter() - .map(log_record_to_pb) - .collect::>>()?; - - Ok(InstrumentationLibraryLogs { - schema_url: ill - .get("schema_url") - .and_then(Value::as_str) - .map(ToString::to_string) - .unwrap_or_default(), - instrumentation_library: ill - .get("instrumentation_library") - .map(instrumentation_library_to_pb) - .transpose()?, - logs, - }) - }) - .collect() -} - pub(crate) fn resource_logs_to_json(request: ExportLogsServiceRequest) -> Result> { let logs = request .resource_logs .into_iter() .map(|log| { - let ill = instrumentation_library_logs_to_json(log.instrumentation_library_logs)?; - - let mut base = - literal!({ "instrumentation_library_logs": ill, "schema_url": log.schema_url}); + let mut base = literal!({ "schema_url": log.schema_url}); if let Some(r) = log.resource { base.try_insert("resource", resource::resource_to_json(r)); }; + base.try_insert("scope_logs", scope_logs_to_json(log.scope_logs)?); Ok(base) }) .collect::>>()?; @@ -166,91 +138,96 @@ pub(crate) fn resource_logs_to_json(request: ExportLogsServiceRequest) -> Result Ok(literal!({ "logs": logs })) } +pub(crate) fn scope_to_pb(json: &Value<'_>) -> Result { + let _json = json + .as_object() + .ok_or("Invalid json mapping for InstrumentationScope")?; + Ok(InstrumentationScope { + name: pb::maybe_string_to_pb(json.get("name")).unwrap_or_default(), + version: pb::maybe_string_to_pb(json.get("version")).unwrap_or_default(), + attributes: Vec::new(), + dropped_attributes_count: 0u32, + }) +} + +pub(crate) fn log_records_to_pb(json: Option<&Value<'_>>) -> Result> { + if let Some(json) = json { + json.as_array() + .ok_or("Invalid json mapping for [LogRecord, ...]")? + .iter() + .map(log_record_to_pb) + .collect() + } else { + Ok(vec![]) + } +} + +pub(crate) fn scope_logs_to_pb(json: Option<&Value<'_>>) -> Result> { + if let Some(json) = json { + json.as_array() + .ok_or("Invalid json mapping for ScopeLogs")? + .iter() + .filter_map(Value::as_object) + .map(|item| { + Ok(ScopeLogs { + scope: item.get("scope").map(scope_to_pb).transpose()?, + log_records: log_records_to_pb(item.get("log_records"))?, + schema_url: item + .get("schema_url") + .and_then(Value::as_str) + .unwrap_or_default() + .to_string(), + }) + }) + .collect() + } else { + Ok(vec![]) + } +} + +pub(crate) fn resources_to_pb(json: Option<&Value<'_>>) -> Result> { + if let Some(json) = json { + json.as_array() + .ok_or("Invalid json mapping for [Resource,...]")? + .iter() + .map(resource_to_pb) + .collect() + } else { + Ok(vec![]) + } +} + pub(crate) fn resource_logs_to_pb(json: &Value<'_>) -> Result> { json.get_array("logs") .ok_or("Missing `logs` array")? .iter() - .filter_map(Value::as_object) .map(|data| { Ok(ResourceLogs { schema_url: data .get("schema_url") .map(ToString::to_string) .unwrap_or_default(), - instrumentation_library_logs: maybe_instrumentation_library_logs_to_pb( - data.get("instrumentation_library_logs"), - )?, - resource: data.get("resource").map(resource_to_pb).transpose()?, + resource: match data.get("resource") { + Some(resource) => Some(resource_to_pb(resource)?), + None => None, + }, + scope_logs: scope_logs_to_pb(data.get("scope_logs"))?, }) }) .collect() } #[cfg(test)] +#[allow(clippy::unnecessary_wraps)] // Don't error highlight this in tests - we do not care mod tests { use tremor_otelapis::opentelemetry::proto::{ - common::v1::{any_value, AnyValue, InstrumentationLibrary}, + common::v1::{any_value, AnyValue}, resource::v1::Resource, }; use tremor_script::utils::sorted_serialize; use super::*; - #[test] - fn instrumentation_library_logs() -> Result<()> { - let nanos = tremor_common::time::nanotime(); - let span_id_pb = id::random_span_id_bytes(nanos); - let span_id_json = id::test::pb_span_id_to_json(&span_id_pb); - let trace_id_json = id::random_trace_id_value(nanos); - let trace_id_pb = id::test::json_trace_id_to_pb(Some(&trace_id_json))?; - - let pb = vec![InstrumentationLibraryLogs { - schema_url: "schema_url".into(), - instrumentation_library: Some(InstrumentationLibrary { - name: "name".into(), - version: "v0.1.2".into(), - }), // TODO For now its an error for this to be None - may need to revisit - logs: vec![LogRecord { - time_unix_nano: 0, - severity_number: 9, - severity_text: "INFO".into(), - name: "test".into(), - body: Some(AnyValue { - value: Some(any_value::Value::StringValue("snot".into())), - }), // TODO For now its an error for this to be None - may need to revisit - attributes: vec![], - dropped_attributes_count: 100, - flags: 128, - span_id: span_id_pb.clone(), - trace_id: trace_id_pb, - }], - }]; - let json = instrumentation_library_logs_to_json(pb.clone())?; - let back_again = maybe_instrumentation_library_logs_to_pb(Some(&json))?; - let expected: Value = literal!([{ - "instrumentation_library": { "name": "name", "version": "v0.1.2" }, - "schema_url": "schema_url", - "logs": [ - { "severity_number": 9, - "flags": 128, - "span_id": span_id_json, - "trace_id": trace_id_json, - "dropped_attributes_count": 100, - "time_unix_nano": 0, - "severity_text": "INFO", - "name": "test", - "attributes": {}, - "body": "snot" - } - ] - }]); - - assert_eq!(expected, json); - assert_eq!(pb, back_again); - - Ok(()) - } - #[test] fn resource_logs() -> Result<()> { let nanos = tremor_common::time::nanotime(); @@ -266,17 +243,19 @@ mod tests { attributes: vec![], dropped_attributes_count: 8, }), - instrumentation_library_logs: vec![InstrumentationLibraryLogs { + scope_logs: vec![ScopeLogs { schema_url: "schema_url".into(), - instrumentation_library: Some(InstrumentationLibrary { - name: "name".into(), - version: "v0.1.2".into(), - }), // TODO For now its an error for this to be None - may need to revisit - logs: vec![LogRecord { + scope: Some(InstrumentationScope { + name: "snot".to_string(), + version: "v1.2.3.4".to_string(), + attributes: vec![], + dropped_attributes_count: 0, + }), + log_records: vec![LogRecord { + observed_time_unix_nano: 0, time_unix_nano: 0, severity_number: 9, severity_text: "INFO".into(), - name: "test".into(), body: Some(AnyValue { value: Some(any_value::Value::StringValue("snot".into())), }), // TODO For now its an error for this to be None - may need to revisit @@ -292,30 +271,34 @@ mod tests { let json = resource_logs_to_json(pb.clone())?; let back_again = resource_logs_to_pb(&json)?; let expected: Value = literal!({ - "logs": [ - { - "resource": { "attributes": {}, "dropped_attributes_count": 8 }, + "logs": [{ + "resource": { + "attributes": {}, + "dropped_attributes_count": 8 + }, + "schema_url": "schema_url", + "scope_logs":[{ + "log_records": [{ + "attributes": {}, + "body": "snot", + "dropped_attributes_count": 100, + "flags": 128, + "observed_time_unix_nano": 0, + "severity_number": 9, + "severity_text": "INFO", + "span_id": span_id_json, + "time_unix_nano": 0, + "trace_id": trace_id_json, + }], "schema_url": "schema_url", - "instrumentation_library_logs": [ - { - "instrumentation_library": { "name": "name", "version": "v0.1.2" }, - "schema_url": "schema_url", - "logs": [{ - "severity_number": 9, - "flags": 128, - "span_id": span_id_json, - "trace_id": trace_id_json, - "dropped_attributes_count": 100, - "time_unix_nano": 0, - "severity_text": "INFO", - "name": "test", - "attributes": {}, - "body": "snot" - }] - } - ] - } - ] + "scope": { + "attributes": {}, + "dropped_attributes_count": 0, + "name": "snot", + "version": "v1.2.3.4" + } + }] + }] }); assert_eq!(sorted_serialize(&expected)?, sorted_serialize(&json)?); @@ -339,25 +322,27 @@ mod tests { attributes: vec![], dropped_attributes_count: 8, }), - instrumentation_library_logs: vec![InstrumentationLibraryLogs { - schema_url: "schema_url".into(), - instrumentation_library: Some(InstrumentationLibrary { - name: "name".into(), - version: "v0.1.2".into(), - }), // TODO For now its an error for this to be None - may need to revisit - logs: vec![LogRecord { + scope_logs: vec![ScopeLogs { + schema_url: "schema_url2".to_string(), + scope: Some(InstrumentationScope { + name: "name".to_string(), + version: "v1.2.3.4".to_string(), + attributes: vec![], + dropped_attributes_count: 0, + }), + log_records: vec![LogRecord { time_unix_nano: 0, - severity_number: 0, - severity_text: String::new(), - name: "test".into(), + observed_time_unix_nano: 0, + flags: 0, body: Some(AnyValue { value: Some(any_value::Value::StringValue("snot".into())), - }), // TODO For now its an error for this to be None - may need to revisit + }), attributes: vec![], - dropped_attributes_count: 100, - flags: 128, + dropped_attributes_count: 0, + severity_number: 0, + severity_text: "not scarey".to_string(), span_id: span_id_pb.clone(), - trace_id: trace_id_pb, + trace_id: trace_id_pb.clone(), }], }], }], @@ -365,30 +350,34 @@ mod tests { let json = resource_logs_to_json(pb.clone())?; let back_again = resource_logs_to_pb(&json)?; let expected: Value = literal!({ - "logs": [ - { - "resource": { "attributes": {}, "dropped_attributes_count": 8 }, - "schema_url": "schema_url", - "instrumentation_library_logs": [ - { - "instrumentation_library": { "name": "name", "version": "v0.1.2" }, - "schema_url": "schema_url", - "logs": [{ - "severity_number": 0, - "flags": 128, - "span_id": span_id_json, - "trace_id": trace_id_json, - "dropped_attributes_count": 100, - "time_unix_nano": 0, - "severity_text": "", - "name": "test", - "attributes": {}, - "body": "snot" - }] - } - ] - } - ] + "logs": [{ + "resource": { + "attributes": {}, + "dropped_attributes_count": 8 + }, + "schema_url": "schema_url", + "scope_logs": [{ + "log_records": [{ + "attributes": {}, + "body": "snot", + "dropped_attributes_count": 0, + "flags": 0, + "observed_time_unix_nano": 0, + "severity_number": 0, + "severity_text": "not scarey", + "span_id": span_id_json, + "time_unix_nano": 0, + "trace_id": trace_id_json, + }], + "scope":{ + "attributes": {}, + "dropped_attributes_count": 0, + "name": "name", + "version": "v1.2.3.4" + }, + "schema_url": "schema_url2", + }] + }] }); assert_eq!(sorted_serialize(&expected)?, sorted_serialize(&json)?); @@ -401,49 +390,30 @@ mod tests { fn minimal_logs() { let log = literal!({"logs": [ { - "instrumentation_library_logs": [ - ], + "scope_logs": [], "schema_url": "" } ] }); assert_eq!( Ok(vec![ResourceLogs { - instrumentation_library_logs: vec![], resource: None, - schema_url: String::new() + schema_url: String::from(""), + scope_logs: vec![], }]), resource_logs_to_pb(&log) ); } - #[test] - fn minimal_instrumentation_library_logs() { - let ill = literal!([ - { - "logs": [], - "schema_url": "" - } - ]); - assert_eq!( - Ok(vec![InstrumentationLibraryLogs { - instrumentation_library: None, - logs: vec![], - schema_url: String::new() - }]), - maybe_instrumentation_library_logs_to_pb(Some(&ill)) - ); - } - #[test] fn minimal_log_record() { let lr = literal!({}); assert_eq!( Ok(LogRecord { + observed_time_unix_nano: 0, time_unix_nano: 0, severity_number: 0, severity_text: String::new(), - name: String::new(), body: None, attributes: vec![], dropped_attributes_count: 0, diff --git a/src/connectors/impls/otel/metrics.rs b/src/connectors/impls/otel/metrics.rs index b25633d353..7e851a0aa1 100644 --- a/src/connectors/impls/otel/metrics.rs +++ b/src/connectors/impls/otel/metrics.rs @@ -19,9 +19,13 @@ use super::{ resource::{self, resource_to_pb}, }; use crate::{ - connectors::utils::pb::{self, maybe_from_value}, + connectors::{ + impls::otel::common::maybe_instrumentation_scope_to_json, + utils::pb::{self}, + }, errors::Result, }; +use simd_json::StaticNode; use tremor_otelapis::opentelemetry::proto::{ collector::metrics::v1::ExportMetricsServiceRequest, metrics::v1::{ @@ -29,59 +33,16 @@ use tremor_otelapis::opentelemetry::proto::{ metric::{self, Data}, number_data_point, summary_data_point::ValueAtQuantile, - Exemplar, Gauge, Histogram, HistogramDataPoint, InstrumentationLibraryMetrics, - IntDataPoint, IntExemplar, IntGauge, IntHistogram, IntHistogramDataPoint, IntSum, Metric, - NumberDataPoint, ResourceMetrics, Sum, Summary, SummaryDataPoint, + Exemplar, Gauge, Histogram, HistogramDataPoint, Metric, NumberDataPoint, ResourceMetrics, + ScopeMetrics, Sum, Summary, SummaryDataPoint, }, }; use tremor_value::{literal, prelude::*, Value}; -pub(crate) fn int_exemplars_to_json(data: Vec) -> Value<'static> { - data.into_iter() - .map(|exemplar| { - literal!({ - "span_id": id::hex_span_id_to_json(&exemplar.span_id), - "trace_id": id::hex_trace_id_to_json(&exemplar.trace_id), - "filtered_labels": common::string_key_value_to_json(exemplar.filtered_labels), - "time_unix_nano": exemplar.time_unix_nano, - "value": exemplar.value - }) - }) - .collect() -} - -pub(crate) fn int_exemplars_to_pb(json: Option<&Value<'_>>) -> Result> { - json.as_array() - .ok_or("Unable to map json value to Exemplars pb")? - .iter() - .map(|data| { - Ok(IntExemplar { - filtered_labels: common::string_key_value_to_pb(data.get("filtered_labels"))?, - time_unix_nano: pb::maybe_int_to_pbu64(data.get("time_unix_nano"))?, - value: pb::maybe_int_to_pbi64(data.get("value"))?, - span_id: id::hex_span_id_to_pb(data.get("span_id"))?, - trace_id: id::hex_trace_id_to_pb(data.get("trace_id"))?, - }) - }) - .collect() -} - -#[allow(deprecated)] // handling depricated fields is required by the PB files -pub(crate) fn double_exemplars_to_json(data: Vec) -> Value<'static> { +pub(crate) fn exemplars_to_json(data: Vec) -> Value<'static> { data.into_iter() .map(|exemplar| { - let mut filtered_attributes = - common::key_value_list_to_json(exemplar.filtered_attributes); - let mut filtered_labels = common::string_key_value_to_json(exemplar.filtered_labels); - if let Some((attributes, labels)) = filtered_attributes - .as_object_mut() - .zip(filtered_labels.as_object_mut()) - { - for (k, v) in labels.drain() { - attributes.insert(k, v); - } - }; - + let filtered_attributes = common::key_value_list_to_json(exemplar.filtered_attributes); let mut r = literal!({ "span_id": id::hex_span_id_to_json(&exemplar.span_id), "trace_id": id::hex_trace_id_to_json(&exemplar.trace_id), @@ -101,31 +62,28 @@ pub(crate) fn double_exemplars_to_json(data: Vec) -> Value<'static> { }) .collect() } -#[allow(deprecated)] // handling depricated fields is required by the PB files -pub(crate) fn double_exemplars_to_pb(json: Option<&Value<'_>>) -> Result> { + +pub(crate) fn exemplars_to_pb(json: Option<&Value<'_>>) -> Result> { json.as_array() .ok_or("Unable to map json value to Exemplars pb")? .iter() .map(|data| { - let filtered_attributes = match ( - data.get_object("filtered_attributes"), - data.get_object("filtered_labels"), - ) { - (None, None) => return Err("missing field `filtered_attributes`".into()), - (Some(a), None) | (None, Some(a)) => common::obj_key_value_list_to_pb(a), - (Some(a), Some(l)) => { - let mut a = common::obj_key_value_list_to_pb(a); - a.append(&mut common::obj_key_value_list_to_pb(l)); - a - } - }; + let filtered_attributes = + if let Some(filtered_attributes) = data.get_object("filtered_attributes") { + common::obj_key_value_list_to_pb(filtered_attributes) + } else { + Vec::new() + }; Ok(Exemplar { filtered_attributes, - filtered_labels: vec![], span_id: id::hex_span_id_to_pb(data.get("span_id"))?, trace_id: id::hex_trace_id_to_pb(data.get("trace_id"))?, time_unix_nano: pb::maybe_int_to_pbu64(data.get("time_unix_nano"))?, - value: maybe_from_value(data.get("value"))?, + value: match data.get("value") { + Some(Value::Static(StaticNode::F64(v))) => Some(exemplar::Value::AsDouble(*v)), + Some(Value::Static(StaticNode::I64(v))) => Some(exemplar::Value::AsInt(*v)), + _ => return Err("Unable to parse Exemplar value error".into()), + }, }) }) .collect() @@ -154,54 +112,16 @@ pub(crate) fn quantile_values_to_pb(json: Option<&Value<'_>>) -> Result) -> Value<'static> { +pub(crate) fn data_points_to_json(pb: Vec) -> Value<'static> { pb.into_iter() .map(|data| { - literal!({ - "value": data.value, - "start_time_unix_nano": data.start_time_unix_nano, - "time_unix_nano": data.time_unix_nano, - "labels": common::string_key_value_to_json(data.labels), - "exemplars": int_exemplars_to_json(data.exemplars), - }) - }) - .collect() -} - -pub(crate) fn int_data_points_to_pb(json: Option<&Value<'_>>) -> Result> { - json.as_array() - .ok_or("Unable to map json value to otel pb IntDataPoint list")? - .iter() - .map(|item| { - Ok(IntDataPoint { - labels: common::string_key_value_to_pb(item.get("labels"))?, - exemplars: int_exemplars_to_pb(item.get("exemplars"))?, - time_unix_nano: pb::maybe_int_to_pbu64(item.get("time_unix_nano"))?, - start_time_unix_nano: pb::maybe_int_to_pbu64(item.get("start_time_unix_nano"))?, - value: pb::maybe_int_to_pbi64(item.get("value"))?, - }) - }) - .collect() -} - -#[allow(deprecated)] // handling depricated fields is required by the PB files -pub(crate) fn double_data_points_to_json(pb: Vec) -> Value<'static> { - pb.into_iter() - .map(|data| { - let mut attributes = common::key_value_list_to_json(data.attributes); - let mut labels = common::string_key_value_to_json(data.labels); - if let Some((attributes, labels)) = - attributes.as_object_mut().zip(labels.as_object_mut()) - { - for (k, v) in labels.drain() { - attributes.insert(k, v); - } - }; + let attributes = common::key_value_list_to_json(data.attributes); let mut r = literal!({ "start_time_unix_nano": data.start_time_unix_nano, "time_unix_nano": data.time_unix_nano, "attributes": attributes, - "exemplars": double_exemplars_to_json(data.exemplars), + "exemplars": exemplars_to_json(data.exemplars), + "flags": data.flags, }); match data.value { Some(number_data_point::Value::AsDouble(v)) => { @@ -217,89 +137,81 @@ pub(crate) fn double_data_points_to_json(pb: Vec) -> Value<'sta .collect() } -#[allow(deprecated)] // handling depricated fields is required by the PB files -pub(crate) fn double_data_points_to_pb(json: Option<&Value<'_>>) -> Result> { +pub(crate) fn data_points_to_pb(json: Option<&Value<'_>>) -> Result> { json.as_array() .ok_or("Unable to map json value to otel pb NumberDataPoint list")? .iter() .map(|data| { - let attributes = common::get_attributes_or_labes(data)?; + let attributes = common::get_attributes(data)?; Ok(NumberDataPoint { - labels: vec![], attributes, - exemplars: double_exemplars_to_pb(data.get("exemplars"))?, + exemplars: exemplars_to_pb(data.get("exemplars"))?, time_unix_nano: pb::maybe_int_to_pbu64(data.get("time_unix_nano"))?, start_time_unix_nano: pb::maybe_int_to_pbu64(data.get("start_time_unix_nano"))?, - value: maybe_from_value(data.get("value"))?, + value: match data.get("value") { + Some(Value::Static(StaticNode::F64(v))) => { + Some(number_data_point::Value::AsDouble(*v)) + } + Some(Value::Static(StaticNode::I64(v))) => { + Some(number_data_point::Value::AsInt(*v)) + } + _ => return Err("Unable to parse Exemplar value error".into()), + }, + flags: pb::maybe_int_to_pbu32(data.get("flags"))?, }) }) .collect() } #[allow(deprecated)] // handling depricated fields is required by the PB files -pub(crate) fn double_histo_data_points_to_json(pb: Vec) -> Value<'static> { +pub(crate) fn histo_data_points_to_json(pb: Vec) -> Value<'static> { pb.into_iter() .map(|point| { - let mut attributes = common::key_value_list_to_json(point.attributes); - let mut labels = common::string_key_value_to_json(point.labels); - if let Some((attributes, labels)) = - attributes.as_object_mut().zip(labels.as_object_mut()) - { - for (k, v) in labels.drain() { - attributes.insert(k, v); - } - }; + let attributes = common::key_value_list_to_json(point.attributes); literal!({ "start_time_unix_nano": point.start_time_unix_nano, "time_unix_nano": point.time_unix_nano, "attributes": attributes, - "exemplars": double_exemplars_to_json(point.exemplars), + "exemplars": exemplars_to_json(point.exemplars), "sum": point.sum, "count": point.count, "explicit_bounds": point.explicit_bounds, "bucket_counts": point.bucket_counts, + "min": point.min, + "max": point.max, + "flags": point.flags, }) }) .collect() } -#[allow(deprecated)] // handling depricated fields is required by the PB files -pub(crate) fn double_histo_data_points_to_pb( - json: Option<&Value<'_>>, -) -> Result> { +pub(crate) fn histo_data_points_to_pb(json: Option<&Value<'_>>) -> Result> { json.as_array() .ok_or("Unable to map json value to otel pb HistogramDataPoint list")? .iter() .map(|data| { - let attributes = common::get_attributes_or_labes(data)?; + let attributes = common::maybe_key_value_list_to_pb(data.get("attributes"))?; Ok(HistogramDataPoint { - labels: vec![], attributes, time_unix_nano: pb::maybe_int_to_pbu64(data.get("time_unix_nano"))?, start_time_unix_nano: pb::maybe_int_to_pbu64(data.get("start_time_unix_nano"))?, - sum: pb::maybe_double_to_pb(data.get("sum"))?, + sum: Some(pb::maybe_double_to_pb(data.get("sum"))?), count: pb::maybe_int_to_pbu64(data.get("count"))?, - exemplars: double_exemplars_to_pb(data.get("exemplars"))?, + exemplars: exemplars_to_pb(data.get("exemplars"))?, explicit_bounds: pb::f64_repeated_to_pb(data.get("explicit_bounds"))?, bucket_counts: pb::u64_repeated_to_pb(data.get("explicit_bounds"))?, + flags: pb::maybe_int_to_pbu32(data.get("flags"))?, + max: pb::maybe_double_to_pb(data.get("max")).ok(), + min: pb::maybe_double_to_pb(data.get("min")).ok(), }) }) .collect() } -#[allow(deprecated)] // handling depricated fields is required by the PB files pub(crate) fn double_summary_data_points_to_json(pb: Vec) -> Value<'static> { pb.into_iter() .map(|point| { - let mut attributes = common::key_value_list_to_json(point.attributes); - let mut labels = common::string_key_value_to_json(point.labels); - if let Some((attributes, labels)) = - attributes.as_object_mut().zip(labels.as_object_mut()) - { - for (k, v) in labels.drain() { - attributes.insert(k, v); - } - }; + let attributes = common::key_value_list_to_json(point.attributes); literal!({ "start_time_unix_nano": point.start_time_unix_nano, "time_unix_nano": point.time_unix_nano, @@ -307,6 +219,7 @@ pub(crate) fn double_summary_data_points_to_json(pb: Vec) -> V "quantile_values": quantile_values_to_json(point.quantile_values), "sum": point.sum, "count": point.count, + "flags": point.flags, }) }) .collect() @@ -320,121 +233,55 @@ pub(crate) fn double_summary_data_points_to_pb( .ok_or("Unable to map json value to otel pb SummaryDataPoint list")? .iter() .map(|data| { - let attributes = common::get_attributes_or_labes(data)?; + let attributes = common::get_attributes(data)?; Ok(SummaryDataPoint { - labels: vec![], attributes, time_unix_nano: pb::maybe_int_to_pbu64(data.get("time_unix_nano"))?, start_time_unix_nano: pb::maybe_int_to_pbu64(data.get("start_time_unix_nano"))?, sum: pb::maybe_double_to_pb(data.get("sum"))?, count: pb::maybe_int_to_pbu64(data.get("count"))?, quantile_values: quantile_values_to_pb(data.get("quantile_values"))?, + flags: pb::maybe_int_to_pbu32(data.get("flags"))?, }) }) .collect() } -pub(crate) fn int_histo_data_points_to_json(pb: Vec) -> Value<'static> { - pb.into_iter() - .map(|points| { - literal!({ - "start_time_unix_nano": points.start_time_unix_nano, - "time_unix_nano": points.time_unix_nano, - "labels": common::string_key_value_to_json(points.labels), - "exemplars": int_exemplars_to_json(points.exemplars), - "sum": points.sum, - "count": points.count, - "explicit_bounds": points.explicit_bounds, - "bucket_counts": points.bucket_counts, - }) - }) - .collect() -} - -pub(crate) fn int_histo_data_points_to_pb( - json: Option<&Value<'_>>, -) -> Result> { - json.as_array() - .ok_or("Unable to map json value to otel pb IntHistogramDataPoint list")? - .iter() - .map(|item| { - Ok(IntHistogramDataPoint { - labels: common::string_key_value_to_pb(item.get("labels"))?, - time_unix_nano: pb::maybe_int_to_pbu64(item.get("time_unix_nano"))?, - start_time_unix_nano: pb::maybe_int_to_pbu64(item.get("start_time_unix_nano"))?, - sum: pb::maybe_int_to_pbi64(item.get("sum"))?, - count: pb::maybe_int_to_pbu64(item.get("count"))?, - exemplars: int_exemplars_to_pb(item.get("exemplars"))?, - explicit_bounds: pb::f64_repeated_to_pb(item.get("explicit_bounds"))?, - bucket_counts: pb::u64_repeated_to_pb(item.get("explicit_bounds"))?, - }) - }) - .collect() -} - -pub(crate) fn int_sum_data_points_to_json(pb: Vec) -> Value<'static> { - int_data_points_to_json(pb) -} - pub(crate) fn metrics_data_to_json(pb: Option) -> Value<'static> { pb.map(|pb| match pb { - Data::IntGauge(data) => literal!({ - "int-gauge": { - "data_points": int_data_points_to_json(data.data_points) - }}), Data::Sum(data) => literal!({ - "double-sum": { + "sum": { "is_monotonic": data.is_monotonic, - "data_points": double_data_points_to_json(data.data_points), + "data_points": data_points_to_json(data.data_points), "aggregation_temporality": data.aggregation_temporality, }}), Data::Gauge(data) => literal!({ - "double-gauge": { - "data_points": double_data_points_to_json(data.data_points), + "gauge": { + "data_points": data_points_to_json(data.data_points), }}), Data::Histogram(data) => literal!({ - "double-histogram": { - "data_points": double_histo_data_points_to_json(data.data_points), + "histogram": { + "data_points": histo_data_points_to_json(data.data_points), "aggregation_temporality": data.aggregation_temporality, }}), + Data::ExponentialHistogram(_data) => literal!({ + "exponential-histogram": { // TODO FIXME complete + } + }), Data::Summary(data) => literal!({ - "double-summary": { + "summary": { "data_points": double_summary_data_points_to_json(data.data_points), }}), - Data::IntHistogram(data) => literal!({ - "int-histogram": { - "data_points": int_histo_data_points_to_json(data.data_points), - "aggregation_temporality": data.aggregation_temporality, - }}), - Data::IntSum(data) => literal!({ - "int-sum": { - "is_monotonic": data.is_monotonic, - "data_points": int_sum_data_points_to_json(data.data_points), - "aggregation_temporality": data.aggregation_temporality, - } - }), }) .unwrap_or_default() } pub(crate) fn metrics_data_to_pb(data: &Value<'_>) -> Result { - if let Some(json) = data.get_object("int-gauge") { - let data_points = int_data_points_to_pb(json.get("data_points"))?; - Ok(metric::Data::IntGauge(IntGauge { data_points })) - } else if let Some(json) = data.get_object("double-gauge") { - let data_points = double_data_points_to_pb(json.get("data_points"))?; + if let Some(json) = data.get_object("gauge") { + let data_points = data_points_to_pb(json.get("data_points"))?; Ok(metric::Data::Gauge(Gauge { data_points })) - } else if let Some(json) = data.get_object("int-sum") { - let data_points = int_data_points_to_pb(json.get("data_points"))?; - let is_monotonic = pb::maybe_bool_to_pb(json.get("is_monotonic"))?; - let aggregation_temporality = pb::maybe_int_to_pbi32(json.get("aggregation_temporality"))?; - Ok(metric::Data::IntSum(IntSum { - data_points, - aggregation_temporality, - is_monotonic, - })) - } else if let Some(json) = data.get_object("double-sum") { - let data_points = double_data_points_to_pb(json.get("data_points"))?; + } else if let Some(json) = data.get_object("sum") { + let data_points = data_points_to_pb(json.get("data_points"))?; let is_monotonic = pb::maybe_bool_to_pb(json.get("is_monotonic"))?; let aggregation_temporality = pb::maybe_int_to_pbi32(json.get("aggregation_temporality"))?; Ok(metric::Data::Sum(Sum { @@ -442,21 +289,14 @@ pub(crate) fn metrics_data_to_pb(data: &Value<'_>) -> Result { aggregation_temporality, is_monotonic, })) - } else if let Some(json) = data.get_object("int-histogram") { - let data_points = int_histo_data_points_to_pb(json.get("data_points"))?; - let aggregation_temporality = pb::maybe_int_to_pbi32(json.get("aggregation_temporality"))?; - Ok(metric::Data::IntHistogram(IntHistogram { - data_points, - aggregation_temporality, - })) - } else if let Some(json) = data.get_object("double-histogram") { - let data_points = double_histo_data_points_to_pb(json.get("data_points"))?; + } else if let Some(json) = data.get_object("histogram") { + let data_points = histo_data_points_to_pb(json.get("data_points"))?; let aggregation_temporality = pb::maybe_int_to_pbi32(json.get("aggregation_temporality"))?; Ok(metric::Data::Histogram(Histogram { data_points, aggregation_temporality, })) - } else if let Some(json) = data.get_object("double-summary") { + } else if let Some(json) = data.get_object("summary") { let data_points = double_summary_data_points_to_pb(json.get("data_points"))?; Ok(metric::Data::Summary(Summary { data_points })) } else { @@ -482,70 +322,73 @@ pub(crate) fn metric_to_pb(metric: &Value) -> Result { }) } -pub(crate) fn instrumentation_library_metrics_to_json<'event>( - pb: Vec, -) -> Value<'event> { - let mut json = Vec::with_capacity(pb.len()); - for data in pb { - let metrics: Value = data.metrics.into_iter().map(metric_to_json).collect(); - let mut e = literal!({ "metrics": metrics, "schema_url": data.schema_url }); - if let Some(il) = data.instrumentation_library { - let il = common::maybe_instrumentation_library_to_json(il); - e.try_insert("instrumentation_library", il); - } - json.push(e); +pub(crate) fn metrics_records_to_pb(json: Option<&Value<'_>>) -> Result> { + if let Some(json) = json { + json.as_array() + .ok_or("Invalid json mapping for [MetricRecord, ...]")? + .iter() + .map(metric_to_pb) + .collect() + } else { + Ok(vec![]) } +} - literal!(json) +pub(crate) fn scope_metrics_to_json(pb: Vec) -> Value<'static> { + Value::Array( + pb.into_iter() + .map(|data| { + literal!({ + "scope": maybe_instrumentation_scope_to_json(data.scope), + "metrics": Value::Array(data.metrics.into_iter().map(metric_to_json).collect()), + "schema_url": data.schema_url, + }) + }) + .collect(), + ) } -pub(crate) fn instrumentation_library_metrics_to_pb( - data: Option<&Value<'_>>, -) -> Result> { - let data = data - .as_array() - .ok_or("Invalid json mapping for InstrumentationLibraryMetrics")?; - let mut pb = Vec::with_capacity(data.len()); - for data in data.iter() { - let mut metrics = Vec::new(); - if let Some(data) = data.get_array("metrics") { - for metric in data { - metrics.push(metric_to_pb(metric)?); - } - } - - let e = InstrumentationLibraryMetrics { - schema_url: data - .get_str("schema_url") - .map(ToString::to_string) - .unwrap_or_default(), - instrumentation_library: data - .get("instrumentation_library") - .map(common::instrumentation_library_to_pb) - .transpose()?, - metrics, - }; - pb.push(e); +pub(crate) fn scope_metrics_to_pb(json: Option<&Value<'_>>) -> Result> { + use crate::connectors::impls::otel::logs::scope_to_pb; + if let Some(json) = json { + json.as_array() + .ok_or("Invalid json mapping for ScopeMetrics")? + .iter() + .filter_map(Value::as_object) + .map(|item| { + Ok(ScopeMetrics { + scope: item.get("scope").map(scope_to_pb).transpose()?, + metrics: metrics_records_to_pb(item.get("metrics"))?, + schema_url: item + .get("schema_url") + .and_then(Value::as_str) + .unwrap_or_default() + .to_string(), + }) + }) + .collect() + } else { + Ok(vec![]) } - Ok(pb) } -pub(crate) fn resource_metrics_to_json(request: ExportMetricsServiceRequest) -> Value<'static> { - let metrics: Value = request +pub(crate) fn resource_metrics_to_json( + request: ExportMetricsServiceRequest, +) -> Result> { + let metrics: Result>> = request .resource_metrics .into_iter() .map(|metric| { - let ill = - instrumentation_library_metrics_to_json(metric.instrumentation_library_metrics); - let mut base = literal!({ "instrumentation_library_metrics": ill, "schema_url": metric.schema_url }); + let mut base = literal!({ "schema_url": metric.schema_url }); if let Some(r) = metric.resource { base.try_insert("resource", resource::resource_to_json(r)); }; - base + base.insert("scope_metrics", scope_metrics_to_json(metric.scope_metrics))?; + Ok(base) }) .collect(); - literal!({ "metrics": metrics }) + Ok(literal!({ "metrics": metrics? })) } pub(crate) fn resource_metrics_to_pb(json: Option<&Value<'_>>) -> Result> { @@ -560,10 +403,8 @@ pub(crate) fn resource_metrics_to_pb(json: Option<&Value<'_>>) -> Result>) -> Result Result<()> { - let pb = vec![IntDataPoint { - value: 42, + let pb = vec![NumberDataPoint { + value: Some(OtelMetricsNumberValue::AsInt(42)), start_time_unix_nano: 0, time_unix_nano: 0, - labels: vec![], + flags: 0, exemplars: vec![], + attributes: vec![], }]; - let json = int_data_points_to_json(pb.clone()); - let back_again = int_data_points_to_pb(Some(&json))?; + let json = data_points_to_json(pb.clone()); + let back_again = data_points_to_pb(Some(&json))?; let expected: Value = literal!([{ "value": 42, "start_time_unix_nano": 0, "time_unix_nano": 0, - "labels": {}, - "exemplars": [] + "flags": 0, + "exemplars": [], + "attributes": {}, }]); assert_eq!(expected, json); assert_eq!(pb, back_again); // Empty - let json = int_data_points_to_json(vec![]); - let back_again = int_data_points_to_pb(Some(&json))?; + let json = data_points_to_json(vec![]); + let back_again = data_points_to_pb(Some(&json))?; let expected: Value = literal!([]); assert_eq!(expected, json); assert_eq!(back_again, vec![]); @@ -714,27 +559,28 @@ mod tests { fn double_data_points() -> Result<()> { let pb = vec![NumberDataPoint { attributes: vec![], - value: maybe_from_value(Some(&Value::from(42.42)))?, + value: Some(OtelMetricsNumberValue::AsDouble(42.42)), start_time_unix_nano: 0, time_unix_nano: 0, - labels: vec![], + flags: 0, exemplars: vec![], }]; - let json = double_data_points_to_json(pb.clone()); - let back_again = double_data_points_to_pb(Some(&json))?; + let json = data_points_to_json(pb.clone()); + let back_again = data_points_to_pb(Some(&json))?; let expected: Value = literal!([{ "value": 42.42, "start_time_unix_nano": 0, "time_unix_nano": 0, "attributes": {}, - "exemplars": [] + "exemplars": [], + "flags": 0, }]); assert_eq!(expected, json); assert_eq!(pb, back_again); // Empty - let json = double_data_points_to_json(vec![]); - let back_again = double_data_points_to_pb(Some(&json))?; + let json = data_points_to_json(vec![]); + let back_again = data_points_to_pb(Some(&json))?; let expected: Value = literal!([]); assert_eq!(expected, json); assert_eq!(back_again, vec![]); @@ -744,34 +590,40 @@ mod tests { #[test] fn int_histo_data_points() -> Result<()> { - let pb = vec![IntHistogramDataPoint { - start_time_unix_nano: 0, - time_unix_nano: 0, - labels: vec![], + let pb = vec![HistogramDataPoint { + start_time_unix_nano: 0u64, + time_unix_nano: 0u64, + flags: 0, exemplars: vec![], - sum: 0, + sum: Some(0.0), count: 0, explicit_bounds: vec![], bucket_counts: vec![], + attributes: vec![], + min: Some(0.0), + max: Some(0.0), }]; - let json = int_histo_data_points_to_json(pb.clone()); - let back_again = int_histo_data_points_to_pb(Some(&json))?; + let json = histo_data_points_to_json(pb.clone()); + let back_again = histo_data_points_to_pb(Some(&json))?; let expected: Value = literal!([{ - "start_time_unix_nano": 0, - "time_unix_nano": 0, - "labels": {}, + "start_time_unix_nano": 0u64, + "time_unix_nano": 0u64, + "flags": 0u64, "exemplars": [], - "sum": 0, - "count": 0, + "sum": 0.0, + "count": 0u64, "explicit_bounds": [], "bucket_counts": [], + "attributes": {}, + "min": 0.0, + "max": 0.0 }]); assert_eq!(expected, json); assert_eq!(pb, back_again); // Empty - let json = int_histo_data_points_to_json(vec![]); - let back_again = int_data_points_to_pb(Some(&json))?; + let json = histo_data_points_to_json(vec![]); + let back_again = histo_data_points_to_pb(Some(&json))?; let expected: Value = literal!([]); assert_eq!(expected, json); assert_eq!(back_again, vec![]); @@ -785,31 +637,36 @@ mod tests { attributes: vec![], start_time_unix_nano: 0, time_unix_nano: 0, - labels: vec![], + flags: 0, exemplars: vec![], - sum: 0.0, + sum: Some(0.0), count: 0, explicit_bounds: vec![], bucket_counts: vec![], + min: Some(0.0), + max: Some(0.0), }]; - let json = double_histo_data_points_to_json(pb.clone()); - let back_again = double_histo_data_points_to_pb(Some(&json))?; + let json = histo_data_points_to_json(pb.clone()); + let back_again = histo_data_points_to_pb(Some(&json))?; let expected: Value = literal!([{ - "start_time_unix_nano": 0, - "time_unix_nano": 0, + "start_time_unix_nano": 0u64, + "time_unix_nano": 0u64, "attributes": {}, "exemplars": [], "sum": 0.0, - "count": 0, + "count": 0u64, "explicit_bounds": [], "bucket_counts": [], + "min": 0.0, + "max": 0.0, + "flags": 0 }]); assert_eq!(expected, json); assert_eq!(pb, back_again); // Empty - let json = double_histo_data_points_to_json(vec![]); - let back_again = double_histo_data_points_to_pb(Some(&json))?; + let json = histo_data_points_to_json(vec![]); + let back_again = histo_data_points_to_pb(Some(&json))?; let expected: Value = literal!([]); assert_eq!(expected, json); assert_eq!(back_again, vec![]); @@ -823,23 +680,24 @@ mod tests { attributes: vec![], start_time_unix_nano: 0, time_unix_nano: 0, - labels: vec![], sum: 0.0, count: 0, quantile_values: vec![ValueAtQuantile { value: 0.1, quantile: 0.2, }], + flags: 0, }]; let json = double_summary_data_points_to_json(pb.clone()); let back_again = double_summary_data_points_to_pb(Some(&json))?; let expected: Value = literal!([{ - "start_time_unix_nano": 0, - "time_unix_nano": 0, + "start_time_unix_nano": 0u64, + "time_unix_nano": 0u64, "attributes": {}, "sum": 0.0, "count": 0, - "quantile_values": [ { "value": 0.1, "quantile": 0.2 }] + "quantile_values": [ { "value": 0.1, "quantile": 0.2 }], + "flags": 0, }]); assert_eq!(expected, json); assert_eq!(pb, back_again); @@ -856,25 +714,27 @@ mod tests { #[test] fn metrics_data_int_gauge() -> Result<()> { - let pb = Some(metric::Data::IntGauge(IntGauge { - data_points: vec![IntDataPoint { - value: 42, + let pb = Some(metric::Data::Gauge(Gauge { + data_points: vec![NumberDataPoint { + value: Some(OtelMetricsNumberValue::AsInt(42)), start_time_unix_nano: 0, time_unix_nano: 0, - labels: vec![], + flags: 0, exemplars: vec![], + attributes: vec![], }], })); let json = metrics_data_to_json(pb.clone()); let back_again = metrics_data_to_pb(&json)?; let expected: Value = literal!({ - "int-gauge": { + "gauge": { "data_points": [{ - "start_time_unix_nano": 0, - "time_unix_nano": 0, - "labels": {}, + "start_time_unix_nano": 0u64, + "time_unix_nano": 0u64, + "flags": 0, "exemplars": [], + "attributes": {}, "value": 42 }] }}); @@ -893,7 +753,7 @@ mod tests { value: maybe_from_value(Some(&Value::from(43.43)))?, start_time_unix_nano: 0, time_unix_nano: 0, - labels: vec![], + flags: 0, exemplars: vec![], }], })); @@ -901,14 +761,15 @@ mod tests { let json = metrics_data_to_json(pb.clone()); let back_again = metrics_data_to_pb(&json)?; let expected: Value = literal!({ - "double-sum": { + "sum": { "is_monotonic": false, - "aggregation_temporality": 0, + "aggregation_temporality": 0i64, "data_points": [{ "start_time_unix_nano": 0, "time_unix_nano": 0, "attributes": {}, "exemplars": [], + "flags": 0, "value": 43.43 }] }}); @@ -925,7 +786,7 @@ mod tests { value: maybe_from_value(Some(&Value::from(43.43)))?, start_time_unix_nano: 0, time_unix_nano: 0, - labels: vec![], + flags: 0, exemplars: vec![], }], })); @@ -933,13 +794,14 @@ mod tests { let json = metrics_data_to_json(pb.clone()); let back_again = metrics_data_to_pb(&json)?; let expected: Value = literal!({ - "double-gauge": { + "gauge": { "data_points": [{ "start_time_unix_nano": 0, "time_unix_nano": 0, "attributes": {}, "exemplars": [], - "value": 43.43 + "value": 43.43, + "flags": 0 }] }}); assert_eq!(expected, json); @@ -955,30 +817,35 @@ mod tests { attributes: vec![], start_time_unix_nano: 0, time_unix_nano: 0, - labels: vec![], + flags: 0, exemplars: vec![], count: 5, - sum: 10.0, + sum: Some(10.0), bucket_counts: vec![], explicit_bounds: vec![], + min: Some(0.1), + max: Some(100.0), }], })); let json = metrics_data_to_json(pb.clone()); let back_again = metrics_data_to_pb(&json)?; let expected: Value = literal!({ - "double-histogram": { - "aggregation_temporality": 0, + "histogram": { "data_points": [{ - "start_time_unix_nano": 0, - "time_unix_nano": 0, + "start_time_unix_nano": 0u64, + "time_unix_nano": 0u64, "attributes": {}, "exemplars": [], "sum": 10.0, - "count": 5, + "count": 5u64, + "explicit_bounds": [], "bucket_counts": [], - "explicit_bounds": [] - }] + "min": 0.1, + "max": 100.0, + "flags": 0 + }], + "aggregation_temporality": 0i64, } }); assert_eq!(expected, json); @@ -993,8 +860,8 @@ mod tests { attributes: vec![], start_time_unix_nano: 0, time_unix_nano: 0, - labels: vec![], count: 0, + flags: 0, sum: 0.0, quantile_values: vec![], }], @@ -1003,14 +870,15 @@ mod tests { let json = metrics_data_to_json(pb.clone()); let back_again = metrics_data_to_pb(&json)?; let expected: Value = literal!({ - "double-summary": { + "summary": { "data_points": [{ - "start_time_unix_nano": 0, - "time_unix_nano": 0, + "start_time_unix_nano": 0u64, + "time_unix_nano": 0u64, "attributes": {}, - "count": 0, + "quantile_values": [], "sum": 0.0, - "quantile_values": [] + "count": 0u64, + "flags": 0, }] }}); assert_eq!(expected, json); @@ -1020,35 +888,41 @@ mod tests { #[test] fn metrics_data_int_histo() -> Result<()> { - let pb = Some(metric::Data::IntHistogram(IntHistogram { + let pb = Some(metric::Data::Histogram(Histogram { aggregation_temporality: 0, - data_points: vec![IntHistogramDataPoint { + data_points: vec![HistogramDataPoint { start_time_unix_nano: 0, time_unix_nano: 0, - labels: vec![], + flags: 0, exemplars: vec![], count: 5, - sum: 10, + sum: Some(10.0), bucket_counts: vec![], explicit_bounds: vec![], + min: Some(0.2), + max: Some(99.1), + attributes: vec![], }], })); let json = metrics_data_to_json(pb.clone()); let back_again = metrics_data_to_pb(&json)?; let expected: Value = literal!({ - "int-histogram": { - "aggregation_temporality": 0, + "histogram": { "data_points": [{ - "start_time_unix_nano": 0, - "time_unix_nano": 0, - "labels": {}, + "start_time_unix_nano": 0u64, + "time_unix_nano": 0u64, + "attributes": {}, "exemplars": [], - "count": 5, - "sum": 10, + "sum": 10f64, + "count": 5u64, + "explicit_bounds": [], "bucket_counts": [], - "explicit_bounds": [] - }] + "min": 0.2, + "max": 99.1, + "flags": 0 + }], + "aggregation_temporality": 0i64 }}); assert_eq!(expected, json); assert_eq!(pb, Some(back_again)); @@ -1057,137 +931,39 @@ mod tests { #[test] fn metrics_data_int_sum() -> Result<()> { - let pb = Some(metric::Data::IntSum(IntSum { + let pb = Some(metric::Data::Sum(Sum { is_monotonic: false, aggregation_temporality: 0, - data_points: vec![IntDataPoint { - value: 4, + data_points: vec![NumberDataPoint { + value: Some(OtelMetricsNumberValue::AsInt(4)), start_time_unix_nano: 0, time_unix_nano: 0, - labels: vec![], + flags: 0, exemplars: vec![], + attributes: vec![], }], })); let json = metrics_data_to_json(pb.clone()); let back_again = metrics_data_to_pb(&json)?; let expected: Value = literal!({ - "int-sum": { + "sum": { "is_monotonic": false, - "aggregation_temporality": 0, "data_points": [{ - "start_time_unix_nano": 0, - "time_unix_nano": 0, - "labels": {}, + "start_time_unix_nano": 0u64, + "time_unix_nano": 0u64, + "attributes": {}, "exemplars": [], + "flags": 0u64, "value": 4 - }] + }], + "aggregation_temporality": 0i64 }}); assert_eq!(expected, json); assert_eq!(pb, Some(back_again)); Ok(()) } - #[test] - fn instrumentation_library_metrics() -> Result<()> { - let pb = vec![InstrumentationLibraryMetrics { - schema_url: "schema_url".into(), - instrumentation_library: Some(InstrumentationLibrary { - name: "name".into(), - version: "v0.1.2".into(), - }), - metrics: vec![Metric { - name: "test".into(), - description: "blah blah blah blah".into(), - unit: "badgerfeet".into(), - data: Some(metric::Data::IntGauge(IntGauge { - data_points: vec![IntDataPoint { - value: 42, - start_time_unix_nano: 0, - time_unix_nano: 0, - labels: vec![], - exemplars: vec![], - }], - })), - }], - }]; - let json = instrumentation_library_metrics_to_json(pb.clone()); - let back_again = instrumentation_library_metrics_to_pb(Some(&json))?; - let expected: Value = literal!([{ - "instrumentation_library": { "name": "name", "version": "v0.1.2" }, - "schema_url": "schema_url", - "metrics": [{ - "name": "test", - "description": "blah blah blah blah", - "unit": "badgerfeet", - "data": { - "int-gauge": { - "data_points": [{ - "start_time_unix_nano": 0, - "time_unix_nano": 0, - "labels": {}, - "exemplars": [], - "value": 42 - }] - } - }, - }] - }]); - - assert_eq!(sorted_serialize(&expected)?, sorted_serialize(&json)?); - assert_eq!(pb, back_again); - - Ok(()) - } - - #[test] - fn instrumentation_library_metrics_nolib() -> Result<()> { - let pb = vec![InstrumentationLibraryMetrics { - schema_url: "schema_url".into(), - instrumentation_library: None, - metrics: vec![Metric { - name: "test".into(), - description: "blah blah blah blah".into(), - unit: "badgerfeet".into(), - data: Some(metric::Data::IntGauge(IntGauge { - data_points: vec![IntDataPoint { - value: 42, - start_time_unix_nano: 0, - time_unix_nano: 0, - labels: vec![], - exemplars: vec![], - }], - })), - }], - }]; - let json = instrumentation_library_metrics_to_json(pb.clone()); - let back_again = instrumentation_library_metrics_to_pb(Some(&json))?; - let expected: Value = literal!([{ - "schema_url": "schema_url", - "metrics": [{ - "name": "test", - "description": "blah blah blah blah", - "unit": "badgerfeet", - "data": { - "int-gauge": { - "data_points": [{ - "start_time_unix_nano": 0, - "time_unix_nano": 0, - "labels": {}, - "exemplars": [], - "value": 42 - }] - } - }, - }] - }]); - - assert_eq!(sorted_serialize(&expected)?, sorted_serialize(&json)?); - assert_eq!(pb, back_again); - - Ok(()) - } - #[test] fn resource_metrics() -> Result<()> { let pb = ExportMetricsServiceRequest { @@ -1197,56 +973,65 @@ mod tests { attributes: vec![], dropped_attributes_count: 8, }), - instrumentation_library_metrics: vec![InstrumentationLibraryMetrics { - schema_url: "schema_url".into(), - instrumentation_library: Some(InstrumentationLibrary { - name: "name".into(), - version: "v0.1.2".into(), - }), // TODO For now its an error for this to be None - may need to revisit + scope_metrics: vec![ScopeMetrics { + scope: Some(InstrumentationScope { + name: "snot".to_string(), + version: "v1.2.3.4".to_string(), + attributes: vec![], + dropped_attributes_count: 0, + }), + schema_url: "snot".to_string(), metrics: vec![Metric { name: "test".into(), description: "blah blah blah blah".into(), unit: "badgerfeet".into(), - data: Some(metric::Data::IntGauge(IntGauge { - data_points: vec![IntDataPoint { - value: 42, + data: Some(metric::Data::Gauge(Gauge { + data_points: vec![NumberDataPoint { + value: Some(OtelMetricsNumberValue::AsInt(42)), start_time_unix_nano: 0, time_unix_nano: 0, - labels: vec![], exemplars: vec![], + attributes: vec![], + flags: 0, }], })), }], }], }], }; - let json = resource_metrics_to_json(pb.clone()); + let json = resource_metrics_to_json(pb.clone())?; let back_again = resource_metrics_to_pb(Some(&json))?; let expected: Value = literal!({ "metrics": [ { "resource": { "attributes": {}, "dropped_attributes_count": 8 }, "schema_url": "schema_url", - "instrumentation_library_metrics": [{ - "instrumentation_library": { "name": "name", "version": "v0.1.2" }, - "schema_url": "schema_url", + "scope_metrics": [{ + "schema_url": "snot", "metrics": [{ "name": "test", "description": "blah blah blah blah", "unit": "badgerfeet", "data": { - "int-gauge": { + "gauge": { "data_points": [{ - "start_time_unix_nano": 0, - "time_unix_nano": 0, - "labels": {}, + "start_time_unix_nano": 0u64, + "time_unix_nano": 0u64, + "attributes": {}, + "flags": 0, "exemplars": [], "value": 42 }] } }, - }] - }] + }], + "scope": { + "name": "snot", + "version": "v1.2.3.4", + "attributes": {}, + "dropped_attributes_count": 0 + } + }] } ] }); @@ -1270,29 +1055,13 @@ mod tests { assert_eq!( Ok(vec![ResourceMetrics { resource: None, - instrumentation_library_metrics: vec![], - schema_url: "bla".to_string() + schema_url: "bla".to_string(), + scope_metrics: vec![], }]), resource_metrics_to_pb(Some(&rm)) ); } - #[test] - fn minimal_instrumentation_library_metrics() { - let ilm = literal!([{ - "metrics": [], - "schema_url": "snot" - }]); - assert_eq!( - Ok(vec![InstrumentationLibraryMetrics { - instrumentation_library: None, - metrics: vec![], - schema_url: "snot".to_string() - }]), - instrumentation_library_metrics_to_pb(Some(&ilm)) - ); - } - #[test] fn minimal_metric() { let metric = literal!({ diff --git a/src/connectors/impls/otel/server.rs b/src/connectors/impls/otel/server.rs index 86d4eabb4d..1aa931e61e 100644 --- a/src/connectors/impls/otel/server.rs +++ b/src/connectors/impls/otel/server.rs @@ -12,15 +12,24 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::{common::OtelDefaults, logs, metrics, trace}; +use super::{ + common::{Compression, OtelDefaults}, + logs, metrics, trace, +}; use crate::connectors::prelude::*; use async_std::channel::{bounded, Receiver, Sender}; use async_std::task::JoinHandle; +use tonic::{codegen::CompressionEncoding, transport::Server as GrpcServer}; use tremor_otelapis::all::{self, OpenTelemetryEvents}; +use tremor_otelapis::opentelemetry::proto::collector::{ + logs::v1::logs_service_server::LogsServiceServer, + metrics::v1::metrics_service_server::MetricsServiceServer, + trace::v1::trace_service_server::TraceServiceServer, +}; + const CONNECTOR_TYPE: &str = "otel_server"; // TODO Consider concurrency cap? - #[derive(Debug, Clone, Deserialize, Default)] #[serde(deny_unknown_fields)] pub(crate) struct Config { @@ -35,15 +44,15 @@ pub(crate) struct Config { /// Enables the metrics service #[serde(default = "default_true")] pub(crate) metrics: bool, + /// Configure grpc compression + #[serde(default = "Default::default")] + pub(crate) compression: Compression, } impl ConfigImpl for Config {} - /// The `OpenTelemetry` client connector pub(crate) struct Server { config: Config, - #[allow(dead_code)] - id: String, origin_uri: EventOriginUri, accept_task: Option>>, tx: Sender, @@ -68,7 +77,7 @@ impl ConnectorBuilder for Builder { async fn build_cfg( &self, - id: &Alias, + _id: &Alias, _: &ConnectorConfig, config: &Value, _kill_switch: &KillSwitch, @@ -82,7 +91,6 @@ impl ConnectorBuilder for Builder { let (tx, rx) = bounded(128); Ok(Box::new(Server { config: Config::new(config)?, - id: id.to_string(), origin_uri, accept_task: None, tx, @@ -135,16 +143,10 @@ impl Connector for Server { previous_handle.cancel().await; } - let tx = self.tx.clone(); - - spawn_task( - ctx.clone(), - async move { Ok(all::make(endpoint, tx).await?) }, - ); + self.serve(ctx, endpoint).await; Ok(true) } } - struct OtelSource { origin_uri: EventOriginUri, config: Config, @@ -156,13 +158,13 @@ impl Source for OtelSource { async fn pull_data(&mut self, pull_id: &mut u64, ctx: &SourceContext) -> Result { let (data, remote) = match self.rx.recv().await? { OpenTelemetryEvents::Metrics(metrics, remote) if self.config.metrics => { - (metrics::resource_metrics_to_json(metrics), remote) + (metrics::resource_metrics_to_json(metrics)?, remote) } OpenTelemetryEvents::Logs(logs, remote) if self.config.logs => { (logs::resource_logs_to_json(logs)?, remote) } OpenTelemetryEvents::Trace(traces, remote) if self.config.trace => { - (trace::resource_spans_to_json(traces), remote) + (trace::resource_spans_to_json(traces)?, remote) } _ => { warn!("{ctx} Source received event when support is disabled. Dropping."); @@ -192,11 +194,48 @@ impl Source for OtelSource { } } +impl Server { + async fn serve(&mut self, ctx: &ConnectorContext, addr: std::net::SocketAddr) { + let trace_server = + TraceServiceServer::new(all::TraceServiceForwarder::with_sender(self.tx.clone())); + + let logs_server = + LogsServiceServer::new(all::LogsServiceForwarder::with_sender(self.tx.clone())); + + let metrics_server = + MetricsServiceServer::new(all::MetricsServiceForwarder::with_sender(self.tx.clone())); + + // set the compression on the server. + let (trace_server, logs_server, metrics_server) = match &self.config.compression { + Compression::Gzip => ( + trace_server + .send_compressed(CompressionEncoding::Gzip) + .accept_compressed(CompressionEncoding::Gzip), + logs_server + .send_compressed(CompressionEncoding::Gzip) + .accept_compressed(CompressionEncoding::Gzip), + metrics_server + .send_compressed(CompressionEncoding::Gzip) + .accept_compressed(CompressionEncoding::Gzip), + ), + + Compression::None => (trace_server, logs_server, metrics_server), + }; + + spawn_task(ctx.clone(), async move { + Ok(GrpcServer::builder() + .add_service(trace_server) + .add_service(logs_server) + .add_service(metrics_server) + .serve(addr) + .await?) + }); + } +} + #[cfg(test)] mod tests { use super::*; - // use env_logger; - // use http_types::Method; #[async_std::test] async fn otel_client_builder() -> Result<()> { diff --git a/src/connectors/impls/otel/trace.rs b/src/connectors/impls/otel/trace.rs index d8f6eab02a..2aaf1e27fc 100644 --- a/src/connectors/impls/otel/trace.rs +++ b/src/connectors/impls/otel/trace.rs @@ -15,36 +15,36 @@ #![allow(dead_code)] use super::{ - common::{self, EMPTY}, + common::{self, maybe_instrumentation_scope_to_pb, EMPTY}, id, resource::{self, resource_to_pb}, }; -use crate::connectors::utils::pb::{ - maybe_int_to_pbi32, maybe_int_to_pbu32, maybe_int_to_pbu64, maybe_string_to_pb, +use crate::connectors::{ + impls::otel::common::maybe_instrumentation_scope_to_json, + utils::pb::{maybe_int_to_pbi32, maybe_int_to_pbu32, maybe_int_to_pbu64, maybe_string_to_pb}, }; use crate::errors::Result; use simd_json::Mutable; +use tremor_otelapis::opentelemetry::proto::trace::v1::ScopeSpans; use tremor_value::literal; use tremor_otelapis::opentelemetry::proto::{ collector::trace::v1::ExportTraceServiceRequest, trace::v1::{ span::{Event, Link}, - InstrumentationLibrarySpans, ResourceSpans, Span, Status, + ResourceSpans, Span, Status, }, }; use tremor_value::Value; use value_trait::ValueAccess; -#[allow(deprecated)] pub(crate) fn status_to_json<'event>(data: Option) -> Value<'event> { data.map_or_else( - || literal!({ "code": 0, "deprecated_code": 0, "message": "status code unset" }), + || literal!({ "code": 0, "message": "status code unset" }), |data| { literal!({ "code": data.code, - "deprecated_code": data.deprecated_code, "message": data.message }) }, @@ -119,11 +119,8 @@ pub(crate) fn status_to_pb(json: Option<&Value<'_>>) -> Result> { .as_object() .ok_or("Unable to map json value to pb trace status")?; - // This is generated code in the pb stub code deriving from otel proto files - #[allow(deprecated)] Ok(Some(Status { code: maybe_int_to_pbi32(json.get("code"))?, - deprecated_code: maybe_int_to_pbi32(json.get("deprecated_code"))?, message: maybe_string_to_pb(json.get("message"))?, })) } @@ -172,29 +169,21 @@ pub(crate) fn span_to_pb(span: &Value<'_>) -> Result { }) } -pub(crate) fn instrumentation_library_spans_to_json( - data: Vec, -) -> Value<'static> { - let mut json: Vec = Vec::with_capacity(data.len()); - for data in data { - let spans: Value = data.spans.into_iter().map(span_to_json).collect(); - - let mut e = literal!({ "spans": spans, "schema_url": data.schema_url }); - if let Some(il) = data.instrumentation_library { - let il = common::maybe_instrumentation_library_to_json(il); - e.try_insert("instrumentation_library", il); - } - json.push(e); - } - - Value::from(json) +pub(crate) fn scope_spans_to_json(data: Vec) -> Vec> { + data.into_iter() + .map(|pb| { + literal!({ + "schema_url": pb.schema_url, + "scope": maybe_instrumentation_scope_to_json(pb.scope), + "spans": pb.spans.into_iter().map(span_to_json).collect::>(), + }) + }) + .collect() } -pub(crate) fn instrumentation_library_spans_to_pb( - data: Option<&Value<'_>>, -) -> Result> { +pub(crate) fn scope_spans_to_pb(data: Option<&Value<'_>>) -> Result> { data.as_array() - .ok_or("Invalid json mapping for InstrumentationLibrarySpans")? + .ok_or("Invalid json mapping for ScopeSpans")? .iter() .filter_map(Value::as_object) .map(|data| { @@ -206,54 +195,50 @@ pub(crate) fn instrumentation_library_spans_to_pb( .map(span_to_pb) .collect::>()?; - Ok(InstrumentationLibrarySpans { - instrumentation_library: data - .get("instrumentation_library") - .map(common::instrumentation_library_to_pb) - .transpose()?, + Ok(ScopeSpans { schema_url: data .get("schema_url") .map(ToString::to_string) .unwrap_or_default(), spans, + scope: maybe_instrumentation_scope_to_pb(data.get("scope"))?, }) }) .collect() } -pub(crate) fn resource_spans_to_json(request: ExportTraceServiceRequest) -> Value<'static> { - let json: Value = request +pub(crate) fn resource_spans_to_json(request: ExportTraceServiceRequest) -> Result> { + let json: Result>> = request .resource_spans .into_iter() .map(|span| { - let ill = instrumentation_library_spans_to_json(span.instrumentation_library_spans); - let mut base = - literal!({ "instrumentation_library_spans": ill, "schema_url": span.schema_url }); + let mut base = literal!({ + "schema_url": span.schema_url, + }); if let Some(r) = span.resource { base.try_insert("resource", resource::resource_to_json(r)); }; - base + base.try_insert("scope_spans", scope_spans_to_json(span.scope_spans)); + Ok(base) }) .collect(); - literal!({ "trace": json }) + Ok(literal!({ "trace": Value::Array(json?) })) } -pub(crate) fn resource_spans_to_pb(json: Option<&Value<'_>>) -> Result> { +pub(crate) fn resource_spans_to_pb(json: &Value<'_>) -> Result> { json.get_array("trace") .ok_or("Invalid json mapping for otel trace message - cannot convert to pb")? .iter() .filter_map(Value::as_object) .map(|json| { Ok(ResourceSpans { - instrumentation_library_spans: instrumentation_library_spans_to_pb( - json.get("instrumentation_library_spans"), - )?, schema_url: json .get("schema_url") .map(ToString::to_string) .unwrap_or_default(), resource: json.get("resource").map(resource_to_pb).transpose()?, + scope_spans: scope_spans_to_pb(json.get("scope_spans"))?, }) }) .collect() @@ -263,25 +248,21 @@ pub(crate) fn resource_spans_to_pb(json: Option<&Value<'_>>) -> Result Result<()> { let pb = Status { - deprecated_code: 0, message: "everything is snot".into(), code: 1, }; let json = status_to_json(Some(pb.clone())); let back_again = status_to_pb(Some(&json))?; - let expected: Value = - literal!({"deprecated_code": 0, "message": "everything is snot", "code": 1}); + let expected: Value = literal!({"message": "everything is snot", "code": 1}); assert_eq!(expected, json); assert_eq!(Some(pb), back_again); @@ -289,10 +270,8 @@ mod tests { // None let json = status_to_json(None); let back_again = status_to_pb(Some(&json))?; - let expected: Value = - literal!({"deprecated_code": 0, "message": "status code unset", "code": 0}); + let expected: Value = literal!({"message": "status code unset", "code": 0}); let pb = Status { - deprecated_code: 0, message: "status code unset".into(), code: 0, }; @@ -379,83 +358,6 @@ mod tests { Ok(()) } - #[test] - #[allow(deprecated)] - fn instrument_library_spans() -> Result<()> { - let nanotime = tremor_common::time::nanotime(); - let parent_span_id_json = id::random_span_id_value(nanotime); - let parent_span_id_pb = id::test::json_span_id_to_pb(Some(&parent_span_id_json))?; - let span_id_pb = id::random_span_id_bytes(nanotime); - let span_id_json = id::test::pb_span_id_to_json(&span_id_pb); - let trace_id_json = id::random_trace_id_value(nanotime); - let trace_id_pb = id::test::json_trace_id_to_pb(Some(&trace_id_json))?; - - let pb = vec![InstrumentationLibrarySpans { - schema_url: "schema_url".into(), - instrumentation_library: Some(InstrumentationLibrary { - name: "name".into(), - version: "v0.1.2".into(), - }), // TODO For now its an error for this to be None - may need to revisit - spans: vec![Span { - start_time_unix_nano: 0, - end_time_unix_nano: 0, - name: "test".into(), - attributes: vec![], - dropped_attributes_count: 100, - trace_state: "snot:badger".into(), - parent_span_id: parent_span_id_pb, - span_id: span_id_pb.clone(), - trace_id: trace_id_pb, - kind: 0, - status: Some(Status { - code: 0, - deprecated_code: 0, - message: "woot".into(), - }), - events: vec![], - dropped_events_count: 11, - links: vec![], - dropped_links_count: 13, - }], - }]; - let json = instrumentation_library_spans_to_json(pb.clone()); - let back_again = instrumentation_library_spans_to_pb(Some(&json))?; - let expected: Value = literal!([{ - "instrumentation_library": { "name": "name", "version": "v0.1.2" }, - "schema_url": "schema_url", - "spans": [{ - "start_time_unix_nano": 0, - "end_time_unix_nano": 0, - "name": "test", - "dropped_attributes_count": 100, - "attributes": {}, - "trace_state": "snot:badger", - "parent_span_id": parent_span_id_json, - "span_id": span_id_json, - "trace_id": trace_id_json, - "kind": 0, - "status": { - "code": 0, - "deprecated_code": 0, - "message": "woot" - }, - "events": [], - "links": [], - "dropped_events_count": 11, - "dropped_links_count": 13, - } - ] - }]); - - assert_eq!(expected, json); - assert_eq!(pb, back_again); - - let invalid = instrumentation_library_spans_to_pb(Some(&literal!("snot"))); - assert!(invalid.is_err()); - - Ok(()) - } - #[test] fn resource_spans() -> Result<()> { let nanotime = tremor_common::time::nanotime(); @@ -466,7 +368,6 @@ mod tests { let trace_id_json = id::random_trace_id_value(nanotime); let trace_id_pb = id::test::json_trace_id_to_pb(Some(&trace_id_json))?; - #[allow(deprecated)] let pb = ExportTraceServiceRequest { resource_spans: vec![ResourceSpans { schema_url: "schema_url".into(), @@ -474,12 +375,13 @@ mod tests { attributes: vec![], dropped_attributes_count: 8, }), - instrumentation_library_spans: vec![InstrumentationLibrarySpans { - schema_url: "schema_url".into(), - instrumentation_library: Some(InstrumentationLibrary { - name: "name".into(), - version: "v0.1.2".into(), - }), // TODO For now its an error for this to be None - may need to revisit + scope_spans: vec![ScopeSpans { + scope: Some(InstrumentationScope { + name: "snot".to_string(), + version: "v1.2.3.4".to_string(), + attributes: vec![], + dropped_attributes_count: 0, + }), spans: vec![Span { start_time_unix_nano: 0, end_time_unix_nano: 0, @@ -493,7 +395,6 @@ mod tests { kind: 0, status: Some(Status { code: 0, - deprecated_code: 0, message: "woot".into(), }), events: vec![], @@ -501,22 +402,23 @@ mod tests { links: vec![], dropped_links_count: 13, }], + schema_url: "schema_url".to_string(), }], }], }; - let json = resource_spans_to_json(pb.clone()); - let back_again = resource_spans_to_pb(Some(&json))?; + let json = resource_spans_to_json(pb.clone())?; + let back_again = resource_spans_to_pb(&json)?; let expected: Value = literal!({ "trace": [ { "resource": { "attributes": {}, "dropped_attributes_count": 8 }, "schema_url": "schema_url", - "instrumentation_library_spans": [{ - "instrumentation_library": { "name": "name", "version": "v0.1.2" }, + "scope_spans": [{ "schema_url": "schema_url", + "scope":{"attributes":{},"dropped_attributes_count":0,"name":"snot","version":"v1.2.3.4"}, "spans": [{ - "start_time_unix_nano": 0, - "end_time_unix_nano": 0, + "start_time_unix_nano": 0u64, + "end_time_unix_nano": 0u64, "name": "test", "dropped_attributes_count": 100, "attributes": {}, @@ -527,7 +429,6 @@ mod tests { "kind": 0, "status": { "code": 0, - "deprecated_code": 0, "message": "woot" }, "events": [], @@ -544,7 +445,7 @@ mod tests { assert_eq!(sorted_serialize(&expected)?, sorted_serialize(&json)?); assert_eq!(pb.resource_spans, back_again); - let invalid = resource_spans_to_pb(Some(&literal!("snot"))); + let invalid = resource_spans_to_pb(&literal!("snot")); assert!(invalid.is_err()); Ok(()) @@ -555,7 +456,7 @@ mod tests { let resource_spans = literal!({ "trace": [ { - "instrumentation_library_spans": [], + "scope_spans": [], "schema_url": "schema_url" } ] @@ -563,31 +464,15 @@ mod tests { assert_eq!( Ok(vec![ResourceSpans { resource: None, - instrumentation_library_spans: vec![], - schema_url: "schema_url".to_string() - }]), - resource_spans_to_pb(Some(&resource_spans)) - ); - } - - #[test] - fn minimal_instrumentation_library_spans() { - let ils = literal!([{ - "spans": [], - "schema_url": "schema_url" - }]); - assert_eq!( - Ok(vec![InstrumentationLibrarySpans { - instrumentation_library: None, - spans: vec![], + // instrumentation_library_spans: vec![], + scope_spans: vec![], schema_url: "schema_url".to_string() }]), - instrumentation_library_spans_to_pb(Some(&ils)) + resource_spans_to_pb(&resource_spans) ); } #[test] - #[allow(clippy::cast_possible_truncation)] fn minimal_span() { let span = literal!({ // hex encoded strings diff --git a/src/connectors/tests.rs b/src/connectors/tests.rs index cd21ab4716..4f0dd69590 100644 --- a/src/connectors/tests.rs +++ b/src/connectors/tests.rs @@ -513,7 +513,7 @@ impl TestPipeline { feature = "http-integration", feature = "ws-integration", feature = "s3-integration", - feature = "gcp-integration" +// feature = "gcp-integration" ))] pub(crate) mod free_port { diff --git a/src/connectors/tests/gpubsub/gpub.rs b/src/connectors/tests/gpubsub/gpub.rs index 20c1b45b84..6a96a1747d 100644 --- a/src/connectors/tests/gpubsub/gpub.rs +++ b/src/connectors/tests/gpubsub/gpub.rs @@ -17,9 +17,11 @@ use crate::connectors::tests::ConnectorHarness; use crate::errors::Result; use crate::instance::State; use async_std::prelude::FutureExt; -use googapis::google::pubsub::v1::publisher_client::PublisherClient; -use googapis::google::pubsub::v1::subscriber_client::SubscriberClient; -use googapis::google::pubsub::v1::{PullRequest, Subscription, Topic}; +use google_api_proto::google::pubsub::v1::publisher_client::PublisherClient; +use google_api_proto::google::pubsub::v1::subscriber_client::SubscriberClient; +use google_api_proto::google::pubsub::v1::PullRequest; +use google_api_proto::google::pubsub::v1::Subscription; +use google_api_proto::google::pubsub::v1::Topic; use serial_test::serial; use std::collections::HashSet; use std::time::Duration; @@ -29,6 +31,7 @@ use tonic::transport::Channel; use tremor_common::ports::IN; use tremor_pipeline::{Event, EventId}; use tremor_value::{literal, Value}; +// use tremor_common::ids::SinkId; #[async_std::test] #[serial(gpubsub)] @@ -126,6 +129,9 @@ async fn simple_publish() -> Result<()> { retry_policy: None, detached: false, topic_message_retention_duration: None, + bigquery_config: None, + enable_exactly_once_delivery: false, + state: Default::default(), }) .await?; @@ -158,7 +164,7 @@ async fn simple_publish() -> Result<()> { for msg in result.into_inner().received_messages { let body = msg.message.unwrap(); - received_messages.insert(String::from_utf8(body.data).unwrap()); + received_messages.insert(String::from_utf8(body.data.to_vec()).unwrap()); } iter_count += 1; @@ -232,6 +238,9 @@ async fn simple_publish_with_timeout() -> Result<()> { retry_policy: None, detached: false, topic_message_retention_duration: None, + bigquery_config: None, + enable_exactly_once_delivery: false, + state: Default::default(), }) .await?; diff --git a/src/connectors/tests/gpubsub/gsub.rs b/src/connectors/tests/gpubsub/gsub.rs index c4960c091f..e6fba6293c 100644 --- a/src/connectors/tests/gpubsub/gsub.rs +++ b/src/connectors/tests/gpubsub/gsub.rs @@ -15,19 +15,22 @@ use crate::connectors::impls::gpubsub::consumer::Builder; use crate::connectors::tests::ConnectorHarness; use crate::errors::Result; -use googapis::google::pubsub::v1::publisher_client::PublisherClient; -use googapis::google::pubsub::v1::subscriber_client::SubscriberClient; -use googapis::google::pubsub::v1::{ - GetSubscriptionRequest, PublishRequest, PubsubMessage, Subscription, Topic, -}; +use google_api_proto::google::pubsub::v1::publisher_client::PublisherClient; +use google_api_proto::google::pubsub::v1::subscriber_client::SubscriberClient; +use google_api_proto::google::pubsub::v1::GetSubscriptionRequest; +use google_api_proto::google::pubsub::v1::PublishRequest; +use google_api_proto::google::pubsub::v1::PubsubMessage; +use google_api_proto::google::pubsub::v1::Subscription; +use google_api_proto::google::pubsub::v1::Topic; use serial_test::serial; -use std::collections::HashMap; +use std::collections::BTreeMap; use testcontainers::clients::Cli; use testcontainers::RunnableImage; use tonic::transport::Channel; use tremor_pipeline::CbAction; use tremor_value::{literal, Value}; use value_trait::ValueAccess; +// use tremor_common::ids::SinkId; #[async_std::test] #[serial(gpubsub)] @@ -81,6 +84,9 @@ async fn create_subscription(endpoint: String, topic: &str, subscription: &str) retry_policy: None, detached: false, topic_message_retention_duration: None, + bigquery_config: None, + enable_exactly_once_delivery: false, + state: Default::default(), }) .await?; // assert the system knows about our subscription now @@ -132,7 +138,7 @@ async fn simple_subscribe() -> Result<()> { harness.wait_for_connected().await?; harness.consume_initial_sink_contraflow().await?; - let mut attributes = HashMap::new(); + let mut attributes = BTreeMap::new(); attributes.insert("a".to_string(), "b".to_string()); let channel = Channel::from_shared(endpoint)?.connect().await?; @@ -141,7 +147,7 @@ async fn simple_subscribe() -> Result<()> { .publish(PublishRequest { topic: "projects/test/topics/test".to_string(), messages: vec![PubsubMessage { - data: Vec::from("abc1".as_bytes()), + data: bytes::Bytes::from("abc1".as_bytes()), attributes, message_id: String::new(), publish_time: None, diff --git a/src/connectors/utils/mime.rs b/src/connectors/utils/mime.rs index 2cdaf290bb..ab5ab8fd16 100644 --- a/src/connectors/utils/mime.rs +++ b/src/connectors/utils/mime.rs @@ -109,6 +109,7 @@ impl Default for MimeCodecMap { #[cfg(test)] mod tests { use super::*; + #[test] fn get_mime_type() { let map = MimeCodecMap::default(); diff --git a/tremor-cli/Cargo.toml b/tremor-cli/Cargo.toml index 0555c326c8..c993cbe96f 100644 --- a/tremor-cli/Cargo.toml +++ b/tremor-cli/Cargo.toml @@ -61,6 +61,10 @@ port_scanner = "0.1" shell-words = "1.1" tch = { version = "*", optional = true } termcolor = "1.1" +simdutf8 = "0.1.4" +lexical = "6.1.1" +itoa = "1.0.5" +ryu = "1.0.12" [[bin]] name = "tremor" diff --git a/tremor-cli/tests/integration/otel-compression/assert.yaml b/tremor-cli/tests/integration/otel-compression/assert.yaml new file mode 100644 index 0000000000..df3ac7d025 --- /dev/null +++ b/tremor-cli/tests/integration/otel-compression/assert.yaml @@ -0,0 +1,5 @@ +status: 0 +name: otel connectors +asserts: + - source: out.log + equals_file: expected.json diff --git a/tremor-cli/tests/integration/otel-compression/config.troy b/tremor-cli/tests/integration/otel-compression/config.troy new file mode 100644 index 0000000000..8949afd0b9 --- /dev/null +++ b/tremor-cli/tests/integration/otel-compression/config.troy @@ -0,0 +1,74 @@ +define flow server +flow + use integration; + use tremor::pipelines; + use tremor::connectors; + + define connector otel_server from otel_server + with + config = { + "url": "127.0.0.1:4317", + "compression": "gzip", + } + end; + + define pipeline instrument + into out, exit + pipeline + # recording + select event from in into out; + + # quiescence + select { "exit": 0, "delay": 10 } from in + where present event.trace[0].instrumentation_library_spans[0].spans[0].attributes.`http.target` + into exit; + end; + + create connector data_out from integration::write_file; + create connector exit from integration::exit; + create connector otels from otel_server; + create connector stdio from connectors::console; + + create pipeline echo from instrument; + create pipeline passthrough from pipelines::passthrough; + + # Echo otel server: -> server -> server_side -> + connect /connector/otels to /pipeline/echo; + connect /connector/otels to /pipeline/passthrough; + + connect /pipeline/passthrough to /connector/stdio; + connect /pipeline/echo to /connector/data_out; + connect /pipeline/echo/exit to /connector/exit; +end; + +define flow client +flow + use integration; + use tremor::pipelines; + + define connector otel_client from otel_client + with + config = { + "url": "127.0.0.1:4317", + "compression": "gzip", + }, + reconnect = { + "retry": { + "interval_ms": 100, + "growth_rate": 2, + "max_retries": 3, + } + } + end; + + create connector data_in from integration::read_file; + create connector otelc from otel_client; + create pipeline replay from pipelines::passthrough; + + # Replay recorded events over otel client to server + connect /connector/data_in to /pipeline/replay; + connect /pipeline/replay to /connector/otelc; +end; + +deploy flow server; +deploy flow client; \ No newline at end of file diff --git a/tremor-cli/tests/integration/otel-compression/expected.json b/tremor-cli/tests/integration/otel-compression/expected.json new file mode 100644 index 0000000000..20c48b4438 --- /dev/null +++ b/tremor-cli/tests/integration/otel-compression/expected.json @@ -0,0 +1,2 @@ +{"trace":[{"instrumentation_library_spans":[{"instrumentation_library":{"name":"io.opentelemetry.jdbc","version":"1.11.0"},"schema_url":"","spans":[{"attributes":{"db.connection_string":"h2:mem:","db.name":"c3d75da4-c7e1-4f8e-9c7c-99b8da7fa791","db.statement":"DROP TABLE vet_specialties IF EXISTS","db.system":"h2","db.user":"sa","thread.id":1,"thread.name":"main"},"dropped_attributes_count":0,"dropped_events_count":0,"dropped_links_count":0,"end_time_unix_nano":1646054197371114000,"events":[],"kind":3,"links":[],"name":"c3d75da4-c7e1-4f8e-9c7c-99b8da7fa791","parent_span_id":"","span_id":"afe8d471fad0ef7c","start_time_unix_nano":1646054197366955000,"status":{"code":0,"deprecated_code":0,"message":""},"trace_id":"52c0d1949a4c908818f9e0d1fcfad17e","trace_state":""}]}],"resource":{"attributes":{"host.arch":"x86_64","host.name":"ALT05209","os.description":"Mac OS X 11.6.2","os.type":"darwin","process.command_line":"/usr/local/Cellar/openjdk@11/11.0.12/libexec/openjdk.jdk/Contents/Home:bin:java -javaagent:./jolokia-jvm-1.7.1.jar=port=8000,host=localhost -javaagent:./opentelemetry-javaagent.jar -Dotel.java.agent.debug=true","process.executable.path":"/usr/local/Cellar/openjdk@11/11.0.12/libexec/openjdk.jdk/Contents/Home:bin:java","process.pid":24855,"process.runtime.description":"Homebrew OpenJDK 64-Bit Server VM 11.0.12+0","process.runtime.name":"OpenJDK Runtime Environment","process.runtime.version":"11.0.12+0","service.name":"javaApp","telemetry.auto.version":"1.11.0","telemetry.sdk.language":"java","telemetry.sdk.name":"opentelemetry","telemetry.sdk.version":"1.11.0"},"dropped_attributes_count":0},"schema_url":"https://opentelemetry.io/schemas/1.8.0"}]} +{"trace":[{"instrumentation_library_spans":[{"instrumentation_library":{"name":"io.opentelemetry.tomcat-7.0","version":"1.11.0"},"schema_url":"","spans":[{"attributes":{"http.flavor":"1.1","http.host":"localhost:8090","http.method":"GET","http.route":"/owners","http.scheme":"http","http.server_name":"localhost","http.status_code":200,"http.target":"/owners?lastName=snot","http.user_agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.109 Safari/537.36","net.peer.ip":"0:0:0:0:0:0:0:1","net.peer.name":"localhost","net.peer.port":60972,"net.transport":"ip_tcp","thread.id":245,"thread.name":"http-nio-8090-exec-7"},"dropped_attributes_count":0,"dropped_events_count":0,"dropped_links_count":0,"end_time_unix_nano":1646054328134855000,"events":[],"kind":2,"links":[],"name":"/owners","parent_span_id":"","span_id":"d647be1d9aa0ca5e","start_time_unix_nano":1646054328042871000,"status":{"code":0,"deprecated_code":0,"message":""},"trace_id":"90b0b8020de77bea3f8587b90585821d","trace_state":""}]}],"resource":{"attributes":{"host.arch":"x86_64","host.name":"ALT05209","os.description":"Mac OS X 11.6.2","os.type":"darwin","process.command_line":"/usr/local/Cellar/openjdk@11/11.0.12/libexec/openjdk.jdk/Contents/Home:bin:java -javaagent:./jolokia-jvm-1.7.1.jar=port=8000,host=localhost -javaagent:./opentelemetry-javaagent.jar -Dotel.java.agent.debug=true","process.executable.path":"/usr/local/Cellar/openjdk@11/11.0.12/libexec/openjdk.jdk/Contents/Home:bin:java","process.pid":24855,"process.runtime.description":"Homebrew OpenJDK 64-Bit Server VM 11.0.12+0","process.runtime.name":"OpenJDK Runtime Environment","process.runtime.version":"11.0.12+0","service.name":"javaApp","telemetry.auto.version":"1.11.0","telemetry.sdk.language":"java","telemetry.sdk.name":"opentelemetry","telemetry.sdk.version":"1.11.0"},"dropped_attributes_count":0},"schema_url":"https://opentelemetry.io/schemas/1.8.0"}]} diff --git a/tremor-cli/tests/integration/otel-compression/in.json b/tremor-cli/tests/integration/otel-compression/in.json new file mode 100644 index 0000000000..0be97a0930 --- /dev/null +++ b/tremor-cli/tests/integration/otel-compression/in.json @@ -0,0 +1,2 @@ +{"trace":[{"instrumentation_library_spans":[{"spans":[{"attributes":{"db.statement":"DROP TABLE vet_specialties IF EXISTS","db.connection_string":"h2:mem:","thread.name":"main","db.system":"h2","thread.id":1,"db.user":"sa","db.name":"c3d75da4-c7e1-4f8e-9c7c-99b8da7fa791"},"events":[],"links":[],"span_id":"afe8d471fad0ef7c","parent_span_id":"","trace_id":"52c0d1949a4c908818f9e0d1fcfad17e","start_time_unix_nano":1646054197366955000,"end_time_unix_nano":1646054197371114000,"trace_state":"","dropped_attributes_count":0,"dropped_events_count":0,"dropped_links_count":0,"status":{"code":0,"deprecated_code":0,"message":""},"kind":3,"name":"c3d75da4-c7e1-4f8e-9c7c-99b8da7fa791"}],"schema_url":"","instrumentation_library":{"name":"io.opentelemetry.jdbc","version":"1.11.0"}}],"schema_url":"https://opentelemetry.io/schemas/1.8.0","resource":{"attributes":{"os.description":"Mac OS X 11.6.2","os.type":"darwin","process.command_line":"/usr/local/Cellar/openjdk@11/11.0.12/libexec/openjdk.jdk/Contents/Home:bin:java -javaagent:./jolokia-jvm-1.7.1.jar=port=8000,host=localhost -javaagent:./opentelemetry-javaagent.jar -Dotel.java.agent.debug=true","telemetry.sdk.language":"java","host.arch":"x86_64","telemetry.sdk.name":"opentelemetry","process.runtime.version":"11.0.12+0","service.name":"javaApp","process.pid":24855,"process.runtime.name":"OpenJDK Runtime Environment","telemetry.sdk.version":"1.11.0","process.runtime.description":"Homebrew OpenJDK 64-Bit Server VM 11.0.12+0","host.name":"ALT05209","process.executable.path":"/usr/local/Cellar/openjdk@11/11.0.12/libexec/openjdk.jdk/Contents/Home:bin:java","telemetry.auto.version":"1.11.0"},"dropped_attributes_count":0}}]} +{"trace":[{"instrumentation_library_spans":[{"spans":[{"attributes":{"net.peer.ip":"0:0:0:0:0:0:0:1","http.target":"/owners?lastName=snot","http.route":"/owners","http.method":"GET","http.server_name":"localhost","thread.id":245,"http.user_agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.109 Safari/537.36","net.peer.name":"localhost","net.transport":"ip_tcp","http.host":"localhost:8090","net.peer.port":60972,"http.flavor":"1.1","thread.name":"http-nio-8090-exec-7","http.status_code":200,"http.scheme":"http"},"events":[],"links":[],"span_id":"d647be1d9aa0ca5e","parent_span_id":"","trace_id":"90b0b8020de77bea3f8587b90585821d","start_time_unix_nano":1646054328042871000,"end_time_unix_nano":1646054328134855000,"trace_state":"","dropped_attributes_count":0,"dropped_events_count":0,"dropped_links_count":0,"status":{"code":0,"deprecated_code":0,"message":""},"kind":2,"name":"/owners"}],"schema_url":"","instrumentation_library":{"name":"io.opentelemetry.tomcat-7.0","version":"1.11.0"}}],"schema_url":"https://opentelemetry.io/schemas/1.8.0","resource":{"attributes":{"os.description":"Mac OS X 11.6.2","os.type":"darwin","process.command_line":"/usr/local/Cellar/openjdk@11/11.0.12/libexec/openjdk.jdk/Contents/Home:bin:java -javaagent:./jolokia-jvm-1.7.1.jar=port=8000,host=localhost -javaagent:./opentelemetry-javaagent.jar -Dotel.java.agent.debug=true","telemetry.sdk.language":"java","host.arch":"x86_64","telemetry.sdk.name":"opentelemetry","process.runtime.version":"11.0.12+0","service.name":"javaApp","process.pid":24855,"process.runtime.name":"OpenJDK Runtime Environment","telemetry.sdk.version":"1.11.0","process.runtime.description":"Homebrew OpenJDK 64-Bit Server VM 11.0.12+0","host.name":"ALT05209","process.executable.path":"/usr/local/Cellar/openjdk@11/11.0.12/libexec/openjdk.jdk/Contents/Home:bin:java","telemetry.auto.version":"1.11.0"},"dropped_attributes_count":0}}]} diff --git a/tremor-cli/tests/integration/otel-compression/tags.yaml b/tremor-cli/tests/integration/otel-compression/tags.yaml new file mode 100644 index 0000000000..22cfd25058 --- /dev/null +++ b/tremor-cli/tests/integration/otel-compression/tags.yaml @@ -0,0 +1,9 @@ +[ + "opentelemetry", + "otel", + "otlp", + "echo", + "connectors", + "connector", + "compression" +]