diff --git a/Cargo.lock b/Cargo.lock index 8558be7f4643b..ebe67b664d88d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -438,6 +438,25 @@ dependencies = [ "num", ] +[[package]] +name = "arrow-flight" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c8b0ba0784d56bc6266b79f5de7a24b47024e7b3a0045d2ad4df3d9b686099f" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-ipc", + "arrow-schema", + "base64 0.22.1", + "bytes 1.11.1", + "futures 0.3.31", + "prost 0.13.5", + "prost-types 0.13.5", + "tonic 0.13.1", +] + [[package]] name = "arrow-ipc" version = "56.2.0" @@ -450,6 +469,8 @@ dependencies = [ "arrow-schema", "arrow-select", "flatbuffers", + "lz4_flex", + "zstd 0.13.2", ] [[package]] @@ -1628,7 +1649,7 @@ dependencies = [ "http-body 0.4.6", "hyper 0.14.32", "itoa", - "matchit", + "matchit 0.7.3", "memchr", "mime", "percent-encoding", @@ -1656,7 +1677,7 @@ dependencies = [ "http-body 1.0.0", "http-body-util", "itoa", - "matchit", + "matchit 0.7.3", "memchr", "mime", "percent-encoding", @@ -1669,6 +1690,31 @@ dependencies = [ "tower-service", ] +[[package]] +name = "axum" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b52af3cb4058c895d37317bb27508dccc8e5f2d39454016b297bf4a400597b8" +dependencies = [ + "axum-core 0.5.6", + "bytes 1.11.1", + "futures-util", + "http 1.3.1", + "http-body 1.0.0", + "http-body-util", + "itoa", + "matchit 0.8.4", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "sync_wrapper 1.0.1", + "tower 0.5.3", + "tower-layer", + "tower-service", +] + [[package]] name = "axum-core" version = "0.3.4" @@ -1706,6 +1752,24 @@ dependencies = [ "tower-service", ] +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes 1.11.1", + "futures-core", + "http 1.3.1", + "http-body 1.0.0", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper 1.0.1", + "tower-layer", + "tower-service", +] + [[package]] name = "azure_core" version = "0.30.1" @@ -3286,6 +3350,34 @@ dependencies = [ "uuid", ] +[[package]] +name = "databricks-zerobus-ingest-sdk" +version = "0.6.0" +source = "git+https://github.com/databricks/zerobus-sdk-rs?rev=326f97e7048fc411165b8ffba7cfef460b04c86a#326f97e7048fc411165b8ffba7cfef460b04c86a" +dependencies = [ + "arrow-array", + "arrow-flight", + "arrow-ipc", + "arrow-schema", + "async-trait", + "futures 0.3.31", + "prost 0.13.5", + "prost-types 0.13.5", + "protoc-bin-vendored", + "reqwest 0.12.28", + "serde", + "serde_json", + "smallvec", + "thiserror 1.0.68", + "tokio", + "tokio-retry", + "tokio-stream", + "tokio-util", + "tonic 0.13.1", + "tonic-build 0.13.1", + "tracing 0.1.44", +] + [[package]] name = "dbl" version = "0.3.2" @@ -6605,6 +6697,12 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + [[package]] name = "matrixmultiply" version = "0.3.8" @@ -8563,6 +8661,70 @@ dependencies = [ "prost 0.13.5", ] +[[package]] +name = "protoc-bin-vendored" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1c381df33c98266b5f08186583660090a4ffa0889e76c7e9a5e175f645a67fa" +dependencies = [ + "protoc-bin-vendored-linux-aarch_64", + "protoc-bin-vendored-linux-ppcle_64", + "protoc-bin-vendored-linux-s390_64", + "protoc-bin-vendored-linux-x86_32", + "protoc-bin-vendored-linux-x86_64", + "protoc-bin-vendored-macos-aarch_64", + "protoc-bin-vendored-macos-x86_64", + "protoc-bin-vendored-win32", +] + +[[package]] +name = "protoc-bin-vendored-linux-aarch_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c350df4d49b5b9e3ca79f7e646fde2377b199e13cfa87320308397e1f37e1a4c" + +[[package]] +name = "protoc-bin-vendored-linux-ppcle_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a55a63e6c7244f19b5c6393f025017eb5d793fd5467823a099740a7a4222440c" + +[[package]] +name = "protoc-bin-vendored-linux-s390_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dba5565db4288e935d5330a07c264a4ee8e4a5b4a4e6f4e83fad824cc32f3b0" + +[[package]] +name = "protoc-bin-vendored-linux-x86_32" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8854774b24ee28b7868cd71dccaae8e02a2365e67a4a87a6cd11ee6cdbdf9cf5" + +[[package]] +name = "protoc-bin-vendored-linux-x86_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b38b07546580df720fa464ce124c4b03630a6fb83e05c336fea2a241df7e5d78" + +[[package]] +name = "protoc-bin-vendored-macos-aarch_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89278a9926ce312e51f1d999fee8825d324d603213344a9a706daa009f1d8092" + +[[package]] +name = "protoc-bin-vendored-macos-x86_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81745feda7ccfb9471d7a4de888f0652e806d5795b61480605d4943176299756" + +[[package]] +name = "protoc-bin-vendored-win32" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95067976aca6421a523e491fce939a3e65249bac4b977adee0ee9771568e8aa3" + [[package]] name = "psl" version = "2.1.22" @@ -11501,6 +11663,7 @@ dependencies = [ "futures-core", "futures-io", "futures-sink", + "futures-util", "pin-project-lite", "slab", "tokio", @@ -11694,6 +11857,37 @@ dependencies = [ "tracing 0.1.44", ] +[[package]] +name = "tonic" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e581ba15a835f4d9ea06c55ab1bd4dce26fc53752c69a04aac00703bfb49ba9" +dependencies = [ + "async-trait", + "axum 0.8.8", + "base64 0.22.1", + "bytes 1.11.1", + "h2 0.4.13", + "http 1.3.1", + "http-body 1.0.0", + "http-body-util", + "hyper 1.7.0", + "hyper-timeout 0.5.1", + "hyper-util", + "percent-encoding", + "pin-project", + "prost 0.13.5", + "rustls-native-certs 0.8.1", + "socket2 0.5.10", + "tokio", + "tokio-rustls 0.26.2", + "tokio-stream", + "tower 0.5.3", + "tower-layer", + "tower-service", + "tracing 0.1.44", +] + [[package]] name = "tonic-build" version = "0.9.2" @@ -11720,6 +11914,20 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "tonic-build" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac6f67be712d12f0b41328db3137e0d0757645d8904b4cb7d51cd9c2279e847" +dependencies = [ + "prettyplease 0.2.15", + "proc-macro2 1.0.106", + "prost-build 0.13.5", + "prost-types 0.13.5", + "quote 1.0.44", + "syn 2.0.117", +] + [[package]] name = "tower" version = "0.4.13" @@ -12529,6 +12737,7 @@ dependencies = [ "criterion", "csv", "databend-client", + "databricks-zerobus-ingest-sdk", "deadpool", "derivative", "dirs-next", diff --git a/Cargo.toml b/Cargo.toml index 6581654a58ea9..c33e41f7bb9d0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -331,6 +331,9 @@ prost = { workspace = true, optional = true } prost-reflect = { workspace = true, optional = true } prost-types = { workspace = true, optional = true } +# Databricks Zerobus +databricks-zerobus-ingest-sdk = { git = "https://github.com/databricks/zerobus-sdk-rs", rev = "326f97e7048fc411165b8ffba7cfef460b04c86a", optional = true } + # GCP goauth = { version = "0.16.0", optional = true } smpl_jwt = { version = "0.8.0", default-features = false, optional = true } @@ -602,7 +605,7 @@ enrichment-tables-mmdb = ["dep:maxminddb"] enrichment-tables-memory = ["dep:evmap", "dep:evmap-derive", "dep:thread_local"] # Codecs -codecs-arrow = ["dep:arrow", "dep:arrow-schema", "vector-lib/arrow"] +codecs-arrow = ["dep:arrow", "dep:arrow-schema", "vector-lib/arrow", "databricks-zerobus-ingest-sdk?/arrow-flight"] codecs-opentelemetry = ["vector-lib/opentelemetry"] codecs-syslog = ["vector-lib/syslog"] @@ -808,6 +811,7 @@ sinks-logs = [ "sinks-clickhouse", "sinks-console", "sinks-databend", + "sinks-databricks-zerobus", "sinks-datadog_events", "sinks-datadog_logs", "sinks-datadog_traces", @@ -876,6 +880,7 @@ sinks-chronicle = [] sinks-clickhouse = ["dep:nom", "dep:rust_decimal", "codecs-arrow"] sinks-console = [] sinks-databend = ["dep:databend-client"] +sinks-databricks-zerobus = ["dep:databricks-zerobus-ingest-sdk", "dep:prost-reflect", "dep:base64"] sinks-datadog_events = [] sinks-datadog_logs = [] sinks-datadog_metrics = ["protobuf-build", "dep:prost", "dep:prost-reflect"] diff --git a/lib/codecs/src/encoding/encoder.rs b/lib/codecs/src/encoding/encoder.rs index 4924dd05447b1..960874f92594f 100644 --- a/lib/codecs/src/encoding/encoder.rs +++ b/lib/codecs/src/encoding/encoder.rs @@ -6,16 +6,32 @@ use vector_core::event::Event; #[cfg(feature = "arrow")] use crate::encoding::ArrowStreamSerializer; use crate::{ - encoding::{Error, Framer, Serializer}, + encoding::{Error, Framer, ProtoBatchSerializer, Serializer}, internal_events::{EncoderFramingError, EncoderSerializeError}, }; +/// The output of a batch encoding operation. +/// +/// Different batch serializers produce different output types: +/// - Arrow serializer produces a `RecordBatch` +/// - Proto serializer produces individual byte buffers per event +#[derive(Debug)] +pub enum BatchOutput { + /// An Arrow RecordBatch containing all events encoded as columnar data. + #[cfg(feature = "arrow")] + Arrow(arrow::record_batch::RecordBatch), + /// A list of individually-serialized records (one per event). + Records(Vec>), +} + /// Serializers that support batch encoding (encoding all events at once). #[derive(Debug, Clone)] pub enum BatchSerializer { /// Arrow IPC stream format serializer. #[cfg(feature = "arrow")] Arrow(ArrowStreamSerializer), + /// Protobuf batch serializer that encodes each event individually. + ProtoBatch(ProtoBatchSerializer), } /// An encoder that encodes batches of events. @@ -35,11 +51,28 @@ impl BatchEncoder { &self.serializer } - /// Get the HTTP content type. - #[cfg(feature = "arrow")] - pub const fn content_type(&self) -> &'static str { + /// Encode a batch of events into a `BatchOutput`. + pub fn encode_batch(&self, events: &[Event]) -> Result { match &self.serializer { - BatchSerializer::Arrow(_) => "application/vnd.apache.arrow.stream", + #[cfg(feature = "arrow")] + BatchSerializer::Arrow(serializer) => { + let record_batch = serializer.encode_to_record_batch(events).map_err(|err| { + use crate::encoding::ArrowEncodingError; + match err { + ArrowEncodingError::NullConstraint { .. } => { + Error::SchemaConstraintViolation(Box::new(err)) + } + _ => Error::SerializingError(Box::new(err)), + } + })?; + Ok(BatchOutput::Arrow(record_batch)) + } + BatchSerializer::ProtoBatch(serializer) => { + let records = serializer + .encode_batch(events) + .map_err(|err| Error::SerializingError(Box::new(err)))?; + Ok(BatchOutput::Records(records)) + } } } } @@ -63,18 +96,19 @@ impl tokio_util::codec::Encoder> for BatchEncoder { } }) } - _ => unreachable!("BatchSerializer cannot be constructed without encode()"), + _ => unreachable!( + "tokio Encoder trait is only used for Arrow; other batch serializers use encode_batch()" + ), } } } -/// An wrapper that supports both framed and batch encoding modes. +/// A wrapper that supports both framed and batch encoding modes. #[derive(Debug, Clone)] pub enum EncoderKind { /// Uses framing to encode individual events Framed(Box>), /// Encodes events in batches without framing - #[cfg(feature = "arrow")] Batch(BatchEncoder), } diff --git a/lib/codecs/src/encoding/format/arrow.rs b/lib/codecs/src/encoding/format/arrow.rs index 236d6292d375e..93dcc60508f22 100644 --- a/lib/codecs/src/encoding/format/arrow.rs +++ b/lib/codecs/src/encoding/format/arrow.rs @@ -92,6 +92,15 @@ pub struct ArrowStreamSerializer { } impl ArrowStreamSerializer { + /// Create a new ArrowStreamSerializer with the given configuration + /// Encode events into a `RecordBatch` without writing to IPC stream format. + pub fn encode_to_record_batch( + &self, + events: &[Event], + ) -> Result { + build_record_batch(self.schema.clone(), events) + } + /// Create a new ArrowStreamSerializer with the given configuration pub fn new(config: ArrowStreamSerializerConfig) -> Result { let schema = config.schema.ok_or(ArrowEncodingError::MissingSchema)?; diff --git a/lib/codecs/src/encoding/format/mod.rs b/lib/codecs/src/encoding/format/mod.rs index 85bc094b26947..95f76a9e77750 100644 --- a/lib/codecs/src/encoding/format/mod.rs +++ b/lib/codecs/src/encoding/format/mod.rs @@ -16,6 +16,7 @@ mod native; mod native_json; #[cfg(feature = "opentelemetry")] mod otlp; +mod proto_batch; mod protobuf; mod raw_message; #[cfg(feature = "syslog")] @@ -39,6 +40,7 @@ pub use native::{NativeSerializer, NativeSerializerConfig}; pub use native_json::{NativeJsonSerializer, NativeJsonSerializerConfig}; #[cfg(feature = "opentelemetry")] pub use otlp::{OtlpSerializer, OtlpSerializerConfig}; +pub use proto_batch::{ProtoBatchEncodingError, ProtoBatchSerializer, ProtoBatchSerializerConfig}; pub use protobuf::{ProtobufSerializer, ProtobufSerializerConfig, ProtobufSerializerOptions}; pub use raw_message::{RawMessageSerializer, RawMessageSerializerConfig}; #[cfg(feature = "syslog")] diff --git a/lib/codecs/src/encoding/format/proto_batch.rs b/lib/codecs/src/encoding/format/proto_batch.rs new file mode 100644 index 0000000000000..f98a5649f9e7b --- /dev/null +++ b/lib/codecs/src/encoding/format/proto_batch.rs @@ -0,0 +1,137 @@ +//! Protobuf batch serializer for encoding events as individual protobuf records. +//! +//! Encodes each event in a batch independently into protobuf bytes, producing +//! a `Vec>` where each element is a single serialized protobuf message. + +use prost_reflect::{MessageDescriptor, prost::Message as _}; +use snafu::Snafu; +use std::sync::Arc; +use vector_config::configurable_component; +use vector_core::{ + config::DataType, + event::{Event, Value}, + schema, +}; +use vrl::protobuf::encode::{Options, encode_message}; + +/// Errors that can occur during protobuf batch encoding +#[derive(Debug, Snafu)] +pub enum ProtoBatchEncodingError { + /// No events provided + #[snafu(display("Cannot encode an empty batch"))] + NoEvents, + + /// Unsupported event type + #[snafu(display("Unsupported event type: only Log and Trace events are supported"))] + UnsupportedEventType, + + /// Protobuf encoding failed + #[snafu(display("Protobuf encoding failed: {}", source))] + EncodingFailed { + /// The underlying encoding error + source: vector_common::Error, + }, + + /// Protobuf prost encoding failed + #[snafu(display("Protobuf prost encoding failed: {}", source))] + ProstEncodingFailed { + /// The underlying prost error + source: prost_reflect::prost::EncodeError, + }, +} + +/// Configuration for protobuf batch serialization +#[configurable_component] +#[derive(Clone, Default)] +pub struct ProtoBatchSerializerConfig { + /// The protobuf message descriptor to use for encoding. + #[serde(skip)] + #[configurable(derived)] + pub descriptor: Option, +} + +impl std::fmt::Debug for ProtoBatchSerializerConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ProtoBatchSerializerConfig") + .field( + "descriptor", + &self.descriptor.as_ref().map(|d| d.full_name().to_string()), + ) + .finish() + } +} + +impl ProtoBatchSerializerConfig { + /// Create a new ProtoBatchSerializerConfig with a message descriptor + pub fn new(descriptor: MessageDescriptor) -> Self { + Self { + descriptor: Some(descriptor), + } + } + + /// The data type of events that are accepted by this serializer. + pub fn input_type(&self) -> DataType { + DataType::Log | DataType::Trace + } + + /// The schema required by the serializer. + pub fn schema_requirement(&self) -> schema::Requirement { + schema::Requirement::empty() + } +} + +/// Protobuf batch serializer that encodes each event into individual protobuf bytes. +#[derive(Clone, Debug)] +pub struct ProtoBatchSerializer { + descriptor: Arc, + options: Options, +} + +impl ProtoBatchSerializer { + /// Create a new ProtoBatchSerializer with the given configuration. + pub fn new(config: ProtoBatchSerializerConfig) -> Result { + let descriptor = config.descriptor.ok_or_else(|| { + vector_common::Error::from("Proto batch serializer requires a message descriptor.") + })?; + + Ok(Self { + descriptor: Arc::new(descriptor), + options: Options { + use_json_names: false, + }, + }) + } + + /// Encode a batch of events into individual protobuf byte buffers. + pub fn encode_batch( + &self, + events: &[Event], + ) -> Result>, ProtoBatchEncodingError> { + if events.is_empty() { + return Err(ProtoBatchEncodingError::NoEvents); + } + + let mut records = Vec::with_capacity(events.len()); + + for event in events { + let dynamic_message = match event { + Event::Log(log) => { + encode_message(&self.descriptor, log.value().clone(), &self.options) + } + Event::Trace(trace) => encode_message( + &self.descriptor, + Value::Object(trace.as_map().clone()), + &self.options, + ), + Event::Metric(_) => { + return Err(ProtoBatchEncodingError::UnsupportedEventType); + } + } + .map_err(|source| ProtoBatchEncodingError::EncodingFailed { source: source.into() })?; + + records.push(dynamic_message.encode_to_vec()); + } + + Ok(records) + } +} diff --git a/lib/codecs/src/encoding/mod.rs b/lib/codecs/src/encoding/mod.rs index 25f88406fbae9..9c5ebc27e11f7 100644 --- a/lib/codecs/src/encoding/mod.rs +++ b/lib/codecs/src/encoding/mod.rs @@ -10,7 +10,7 @@ pub mod serializer; mod transformer; pub use chunking::{Chunker, Chunking, GelfChunker}; pub use config::{EncodingConfig, EncodingConfigWithFraming, SinkType}; -pub use encoder::{BatchEncoder, BatchSerializer, Encoder, EncoderKind}; +pub use encoder::{BatchEncoder, BatchOutput, BatchSerializer, Encoder, EncoderKind}; #[cfg(feature = "arrow")] pub use format::{ ArrowEncodingError, ArrowStreamSerializer, ArrowStreamSerializerConfig, SchemaProvider, @@ -21,7 +21,8 @@ pub use format::{ CefSerializerConfig, CsvSerializer, CsvSerializerConfig, GelfSerializer, GelfSerializerConfig, JsonSerializer, JsonSerializerConfig, JsonSerializerOptions, LogfmtSerializer, LogfmtSerializerConfig, NativeJsonSerializer, NativeJsonSerializerConfig, NativeSerializer, - NativeSerializerConfig, ProtobufSerializer, ProtobufSerializerConfig, + NativeSerializerConfig, ProtoBatchEncodingError, ProtoBatchSerializer, + ProtoBatchSerializerConfig, ProtobufSerializer, ProtobufSerializerConfig, ProtobufSerializerOptions, RawMessageSerializer, RawMessageSerializerConfig, TextSerializer, TextSerializerConfig, }; @@ -36,9 +37,7 @@ pub use framing::{ NewlineDelimitedEncoderConfig, VarintLengthDelimitedEncoder, VarintLengthDelimitedEncoderConfig, }; -#[cfg(feature = "arrow")] -pub use serializer::BatchSerializerConfig; -pub use serializer::{Serializer, SerializerConfig}; +pub use serializer::{BatchSerializerConfig, Serializer, SerializerConfig}; pub use transformer::{TimestampFormat, Transformer}; /// An error that occurred while building an encoder. diff --git a/lib/codecs/src/encoding/serializer.rs b/lib/codecs/src/encoding/serializer.rs index 536f836ad8163..40c25398aa694 100644 --- a/lib/codecs/src/encoding/serializer.rs +++ b/lib/codecs/src/encoding/serializer.rs @@ -6,6 +6,7 @@ use vector_core::{config::DataType, event::Event, schema}; #[cfg(feature = "arrow")] use super::format::{ArrowStreamSerializer, ArrowStreamSerializerConfig}; +use super::format::{ProtoBatchSerializer, ProtoBatchSerializerConfig}; #[cfg(feature = "opentelemetry")] use super::format::{OtlpSerializer, OtlpSerializerConfig}; #[cfg(feature = "syslog")] @@ -160,17 +161,31 @@ pub enum BatchSerializerConfig { #[cfg(feature = "arrow")] #[serde(rename = "arrow_stream")] ArrowStream(ArrowStreamSerializerConfig), + + /// Encodes each event individually as a [Protocol Buffers][protobuf] message. + /// + /// Each event in the batch is serialized to protobuf bytes independently, + /// producing a list of byte buffers (one per event). + /// + /// [protobuf]: https://protobuf.dev/ + #[serde(rename = "proto_batch")] + ProtoBatch(ProtoBatchSerializerConfig), } -#[cfg(feature = "arrow")] impl BatchSerializerConfig { - /// Build the `ArrowStreamSerializer` from this configuration. + /// Build the `BatchSerializer` from this configuration. pub fn build( &self, - ) -> Result> { + ) -> Result> { match self { + #[cfg(feature = "arrow")] BatchSerializerConfig::ArrowStream(arrow_config) => { - Ok(ArrowStreamSerializer::new(arrow_config.clone())?) + let serializer = ArrowStreamSerializer::new(arrow_config.clone())?; + Ok(super::BatchSerializer::Arrow(serializer)) + } + BatchSerializerConfig::ProtoBatch(proto_config) => { + let serializer = ProtoBatchSerializer::new(proto_config.clone())?; + Ok(super::BatchSerializer::ProtoBatch(serializer)) } } } @@ -178,14 +193,18 @@ impl BatchSerializerConfig { /// The data type of events that are accepted by this batch serializer. pub fn input_type(&self) -> DataType { match self { + #[cfg(feature = "arrow")] BatchSerializerConfig::ArrowStream(arrow_config) => arrow_config.input_type(), + BatchSerializerConfig::ProtoBatch(proto_config) => proto_config.input_type(), } } /// The schema required by the batch serializer. pub fn schema_requirement(&self) -> schema::Requirement { match self { + #[cfg(feature = "arrow")] BatchSerializerConfig::ArrowStream(arrow_config) => arrow_config.schema_requirement(), + BatchSerializerConfig::ProtoBatch(proto_config) => proto_config.schema_requirement(), } } } diff --git a/src/sinks/clickhouse/config.rs b/src/sinks/clickhouse/config.rs index 827b684f2234b..a68a30c707902 100644 --- a/src/sinks/clickhouse/config.rs +++ b/src/sinks/clickhouse/config.rs @@ -280,7 +280,7 @@ impl ClickhouseConfig { }; if let Some(batch_encoding) = &self.batch_encoding { - use vector_lib::codecs::{BatchEncoder, BatchSerializer}; + use vector_lib::codecs::BatchEncoder; // Validate that batch_encoding is only compatible with ArrowStream format if self.format != Format::ArrowStream { @@ -293,6 +293,11 @@ impl ClickhouseConfig { let mut arrow_config = match batch_encoding { BatchSerializerConfig::ArrowStream(config) => config.clone(), + _ => { + return Err( + "'batch_encoding' for ClickHouse must use 'arrow_stream' codec.".into(), + ); + } }; self.resolve_arrow_schema( @@ -305,8 +310,7 @@ impl ClickhouseConfig { .await?; let resolved_batch_config = BatchSerializerConfig::ArrowStream(arrow_config); - let arrow_serializer = resolved_batch_config.build()?; - let batch_serializer = BatchSerializer::Arrow(arrow_serializer); + let batch_serializer = resolved_batch_config.build()?; let encoder = EncoderKind::Batch(BatchEncoder::new(batch_serializer)); return Ok((Format::ArrowStream, encoder)); diff --git a/src/sinks/databricks_zerobus/config.rs b/src/sinks/databricks_zerobus/config.rs new file mode 100644 index 0000000000000..78718e6016eb9 --- /dev/null +++ b/src/sinks/databricks_zerobus/config.rs @@ -0,0 +1,500 @@ +//! Configuration for the Zerobus sink. + +use vector_lib::configurable::configurable_component; +use vector_lib::sensitive_string::SensitiveString; + +use crate::config::{AcknowledgementsConfig, GenerateConfig, Input, SinkConfig, SinkContext}; +use crate::sinks::{ + prelude::*, + util::{BatchConfig, RealtimeSizeBasedDefaultBatchSettings}, +}; + +use vector_lib::codecs::encoding::{ + BatchEncoder, BatchSerializerConfig, ProtoBatchSerializerConfig, +}; + +use super::{ + error::ZerobusSinkError, + service::{StreamMode, ZerobusService}, + sink::ZerobusSink, +}; + +/// Authentication configuration for Databricks. +#[configurable_component] +#[derive(Clone, Debug)] +#[serde(tag = "strategy", rename_all = "snake_case")] +pub enum DatabricksAuthentication { + /// Authenticate using OAuth 2.0 client credentials. + #[serde(rename = "oauth")] + OAuth { + /// OAuth 2.0 client ID. + #[configurable(metadata(docs::examples = "${DATABRICKS_CLIENT_ID}"))] + #[configurable(metadata(docs::examples = "abc123..."))] + client_id: SensitiveString, + + /// OAuth 2.0 client secret. + #[configurable(metadata(docs::examples = "${DATABRICKS_CLIENT_SECRET}"))] + #[configurable(metadata(docs::examples = "secret123..."))] + client_secret: SensitiveString, + }, +} + +/// Schema source configuration for defining table schema. +/// +/// The schema can be provided as either: +/// - A path to a protobuf descriptor file (.desc or .pb) +/// - Dynamically fetched from the Unity Catalog table +#[configurable_component] +#[derive(Clone, Debug)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum SchemaSource { + /// Path to a protobuf descriptor file. + /// + /// The file should contain a serialized FileDescriptorSet or DescriptorProto. + /// These files are typically generated by protoc with the --descriptor_set_out flag. + /// + /// Example: + /// ```sh + /// protoc --descriptor_set_out=schema.desc --include_imports your_schema.proto + /// ``` + Path { + /// Path to the descriptor file. + #[configurable(metadata(docs::examples = "/path/to/schema.desc"))] + #[configurable(metadata(docs::examples = "./schema.pb"))] + path: String, + + /// The message type to use from the descriptor file. + #[configurable(metadata(docs::examples = "package.Message"))] + message_type: String, + }, + + /// Dynamically fetch schema from Unity Catalog table. + /// + /// This will query the Unity Catalog API to get the table schema and + /// automatically generate a protobuf descriptor from it. + /// Uses the same authentication credentials as the sink. + #[serde(rename = "unity_catalog")] + UnityCatalog, +} + +/// Zerobus stream configuration options. +/// +/// This is a thin wrapper around the SDK's `StreamConfigurationOptions` with Vector-specific +/// configuration attributes and custom defaults suitable for Vector's use case. +#[configurable_component] +#[derive(Clone, Debug)] +#[serde(deny_unknown_fields)] +pub struct ZerobusStreamOptions { + /// Timeout in milliseconds for flush operations. + #[serde(default = "default_flush_timeout_ms")] + #[configurable(metadata(docs::examples = 30000))] + pub flush_timeout_ms: u64, + + /// Timeout in milliseconds for server acknowledgments. + #[serde(default = "default_server_ack_timeout_ms")] + #[configurable(metadata(docs::examples = 60000))] + pub server_lack_of_ack_timeout_ms: u64, +} + +impl Default for ZerobusStreamOptions { + fn default() -> Self { + Self { + flush_timeout_ms: default_flush_timeout_ms(), + server_lack_of_ack_timeout_ms: default_server_ack_timeout_ms(), + } + } +} + +impl From for databricks_zerobus_ingest_sdk::StreamConfigurationOptions { + fn from(options: ZerobusStreamOptions) -> Self { + Self { + recovery: true, + recovery_retries: 4, + server_lack_of_ack_timeout_ms: options.server_lack_of_ack_timeout_ms, + flush_timeout_ms: options.flush_timeout_ms, + ..Default::default() + } + } +} + +#[cfg(feature = "codecs-arrow")] +impl From for databricks_zerobus_ingest_sdk::ArrowStreamConfigurationOptions { + fn from(options: ZerobusStreamOptions) -> Self { + Self { + recovery: true, + recovery_retries: 4, + server_lack_of_ack_timeout_ms: options.server_lack_of_ack_timeout_ms, + flush_timeout_ms: options.flush_timeout_ms, + ..Default::default() + } + } +} + +/// Configuration for the Databricks Zerobus sink. +#[configurable_component(sink( + "databricks_zerobus", + "Stream observability data to Databricks Unity Catalog via Zerobus." +))] +#[derive(Clone, Debug)] +#[serde(deny_unknown_fields)] +pub struct ZerobusSinkConfig { + /// The Zerobus ingestion endpoint URL. + /// + /// This should be the full URL to the Zerobus ingestion service. + #[configurable(metadata(docs::examples = "https://ingest.dev.databricks.com"))] + #[configurable(metadata(docs::examples = "https://ingest.prod.databricks.com"))] + pub ingestion_endpoint: String, + + /// The Unity Catalog table name to write to. + /// + /// This should be in the format `catalog.schema.table`. + #[configurable(metadata(docs::examples = "logging_platform.my_team.logs"))] + #[configurable(metadata(docs::examples = "main.default.vector_logs"))] + pub table_name: String, + + /// The Unity Catalog endpoint URL. + /// + /// This is used for authentication and table metadata. + #[configurable(metadata( + docs::examples = "https://dbc-e2f0eb31-2b0e.staging.cloud.databricks.com" + ))] + #[configurable(metadata(docs::examples = "https://your-workspace.cloud.databricks.com"))] + pub unity_catalog_endpoint: String, + + /// Databricks authentication configuration. + #[configurable(derived)] + pub auth: DatabricksAuthentication, + + /// Schema definition for the table. + /// + /// The schema must be provided either as: + /// - A path to a protobuf descriptor file + /// - Unity Catalog table schema (fetched automatically) + /// + /// Protobuf descriptors can be generated using protoc: + /// ```sh + /// protoc --descriptor_set_out=schema.desc --include_imports your_schema.proto + /// ``` + #[configurable(derived)] + pub schema: SchemaSource, + + /// Zerobus stream configuration options. + #[serde(default)] + pub stream_options: ZerobusStreamOptions, + + /// The batch encoding configuration for encoding events in batches. + /// + /// Defaults to protobuf batch encoding, which serializes each event + /// individually as a protobuf message. + #[configurable(derived)] + #[serde(default = "default_batch_encoding")] + pub batch_encoding: BatchSerializerConfig, + + #[configurable(derived)] + #[serde(default)] + pub batch: BatchConfig, + + #[configurable(derived)] + #[serde(default)] + pub request: TowerRequestConfig, + + #[configurable(derived)] + #[serde( + default, + deserialize_with = "crate::serde::bool_or_struct", + skip_serializing_if = "crate::serde::is_default" + )] + pub acknowledgements: AcknowledgementsConfig, +} + +impl GenerateConfig for ZerobusSinkConfig { + fn generate_config() -> toml::Value { + toml::Value::try_from(Self { + ingestion_endpoint: "https://ingest.dev.databricks.com".to_string(), + table_name: "catalog.schema.table".to_string(), + unity_catalog_endpoint: "https://your-workspace.cloud.databricks.com".to_string(), + auth: DatabricksAuthentication::OAuth { + client_id: SensitiveString::from("${DATABRICKS_CLIENT_ID}".to_string()), + client_secret: SensitiveString::from("${DATABRICKS_CLIENT_SECRET}".to_string()), + }, + schema: SchemaSource::UnityCatalog, + stream_options: ZerobusStreamOptions::default(), + batch_encoding: default_batch_encoding(), + batch: BatchConfig::default(), + request: TowerRequestConfig::default(), + acknowledgements: AcknowledgementsConfig::default(), + }) + .unwrap() + } +} + +#[async_trait::async_trait] +#[typetag::serde(name = "databricks_zerobus")] +impl SinkConfig for ZerobusSinkConfig { + async fn build(&self, _cx: SinkContext) -> crate::Result<(VectorSink, Healthcheck)> { + let descriptor = ZerobusService::resolve_descriptor(self).await?; + + let mut batch_encoding = self.batch_encoding.clone(); + let stream_mode = match &mut batch_encoding { + BatchSerializerConfig::ProtoBatch(config) => { + config.descriptor = Some(descriptor.clone()); + StreamMode::Proto { + descriptor_proto: std::sync::Arc::new(descriptor.descriptor_proto().clone()), + } + } + #[cfg(feature = "codecs-arrow")] + BatchSerializerConfig::ArrowStream(arrow_config) => { + let arrow_schema = + super::proto_to_arrow::proto_descriptor_to_arrow_schema(&descriptor)?; + arrow_config.schema = Some(arrow_schema.clone()); + StreamMode::Arrow { + arrow_schema: std::sync::Arc::new(arrow_schema), + } + } + }; + let batch_serializer = batch_encoding + .build() + .map_err(|e| format!("Failed to build batch serializer: {}", e))?; + let encoder = BatchEncoder::new(batch_serializer); + + let service = + ZerobusService::new(self.clone(), stream_mode, self.acknowledgements.enabled()).await?; + let healthcheck_service = service.clone(); + + let request_limits = self.request.into_settings(); + + let sink = ZerobusSink::new(service, request_limits, self.batch, encoder)?; + + let healthcheck = async move { + healthcheck_service + .ensure_stream() + .await + .map_err(|e| e.into()) + }; + + Ok(( + VectorSink::from_event_streamsink(sink), + Box::pin(healthcheck), + )) + } + + fn input(&self) -> Input { + Input::log() + } + + fn acknowledgements(&self) -> &AcknowledgementsConfig { + &self.acknowledgements + } +} + +impl ZerobusSinkConfig { + pub fn validate(&self) -> Result<(), ZerobusSinkError> { + if self.ingestion_endpoint.is_empty() { + return Err(ZerobusSinkError::ConfigError { + message: "ingestion_endpoint cannot be empty".to_string(), + }); + } + + if self.table_name.is_empty() { + return Err(ZerobusSinkError::ConfigError { + message: "table_name cannot be empty".to_string(), + }); + } + + if !self.table_name.contains('.') { + return Err(ZerobusSinkError::ConfigError { + message: "table_name must be in format 'catalog.schema.table'".to_string(), + }); + } + + if self.unity_catalog_endpoint.is_empty() { + return Err(ZerobusSinkError::ConfigError { + message: "unity_catalog_endpoint cannot be empty".to_string(), + }); + } + + // Validate authentication credentials + match &self.auth { + DatabricksAuthentication::OAuth { + client_id, + client_secret, + } => { + if client_id.inner().is_empty() { + return Err(ZerobusSinkError::ConfigError { + message: "OAuth client_id cannot be empty".to_string(), + }); + } + if client_secret.inner().is_empty() { + return Err(ZerobusSinkError::ConfigError { + message: "OAuth client_secret cannot be empty".to_string(), + }); + } + } + } + + if let Some(max_bytes) = self.batch.max_bytes { + // Zerobus SDK limits max bytes to 10MB. + // NOTE: The size of the batch in Vector is not exactly the same as the size of the + // batch in the SDK since they are encoded differently. Though it is expected that + // Vector encoded data will be larger than the SDK encoded data since SDK encodes the + // data in protobuf format. + if max_bytes > 10_000_000 { + return Err(ZerobusSinkError::ConfigError { + message: "max_bytes must be less than or equal to 10MB".to_string(), + }); + } + } + + Ok(()) + } +} + +// Default value functions +const fn default_flush_timeout_ms() -> u64 { + 30000 +} + +const fn default_server_ack_timeout_ms() -> u64 { + 60000 +} + +fn default_batch_encoding() -> BatchSerializerConfig { + BatchSerializerConfig::ProtoBatch(ProtoBatchSerializerConfig::default()) +} + +#[cfg(test)] +mod tests { + use super::*; + use vector_lib::sensitive_string::SensitiveString; + + fn create_test_config() -> ZerobusSinkConfig { + ZerobusSinkConfig { + ingestion_endpoint: "https://test.databricks.com".to_string(), + table_name: "test.default.logs".to_string(), + unity_catalog_endpoint: "https://test-workspace.databricks.com".to_string(), + auth: DatabricksAuthentication::OAuth { + client_id: SensitiveString::from("test-client-id".to_string()), + client_secret: SensitiveString::from("test-client-secret".to_string()), + }, + schema: SchemaSource::UnityCatalog, + stream_options: ZerobusStreamOptions::default(), + batch_encoding: default_batch_encoding(), + batch: Default::default(), + request: Default::default(), + acknowledgements: Default::default(), + } + } + + #[test] + fn test_config_validation_success() { + let config = create_test_config(); + assert!(config.validate().is_ok()); + } + + #[test] + fn test_config_validation_empty_endpoint() { + let mut config = create_test_config(); + config.ingestion_endpoint = "".to_string(); + + let result = config.validate(); + assert!(result.is_err()); + + if let Err(crate::sinks::databricks_zerobus::error::ZerobusSinkError::ConfigError { + message, + }) = result + { + assert!(message.contains("ingestion_endpoint cannot be empty")); + } else { + panic!("Expected ConfigError for empty ingestion_endpoint"); + } + } + + #[test] + fn test_config_validation_empty_table_name() { + let mut config = create_test_config(); + config.table_name = "".to_string(); + + let result = config.validate(); + assert!(result.is_err()); + + if let Err(crate::sinks::databricks_zerobus::error::ZerobusSinkError::ConfigError { + message, + }) = result + { + assert!(message.contains("table_name cannot be empty")); + } else { + panic!("Expected ConfigError for empty table_name"); + } + } + + #[test] + fn test_config_validation_invalid_table_name() { + let mut config = create_test_config(); + config.table_name = "invalid_table".to_string(); // Missing dots + + let result = config.validate(); + assert!(result.is_err()); + + if let Err(crate::sinks::databricks_zerobus::error::ZerobusSinkError::ConfigError { + message, + }) = result + { + assert!(message.contains("catalog.schema.table")); + } else { + panic!("Expected ConfigError for invalid table_name format"); + } + } + + #[test] + fn test_config_validation_empty_unity_catalog_endpoint() { + let mut config = create_test_config(); + config.unity_catalog_endpoint = "".to_string(); + + let result = config.validate(); + assert!(result.is_err()); + + if let Err(crate::sinks::databricks_zerobus::error::ZerobusSinkError::ConfigError { + message, + }) = result + { + assert!(message.contains("unity_catalog_endpoint cannot be empty")); + } else { + panic!("Expected ConfigError for empty unity_catalog_endpoint"); + } + } + + #[test] + fn test_config_validation_empty_oauth_credentials() { + let mut config = create_test_config(); + config.auth = DatabricksAuthentication::OAuth { + client_id: SensitiveString::from("".to_string()), + client_secret: SensitiveString::from("test-secret".to_string()), + }; + + let result = config.validate(); + assert!(result.is_err()); + + if let Err(crate::sinks::databricks_zerobus::error::ZerobusSinkError::ConfigError { + message, + }) = result + { + assert!(message.contains("OAuth client_id cannot be empty")); + } else { + panic!("Expected ConfigError for empty OAuth client_id"); + } + } + + #[test] + fn test_stream_options_conversion() { + let options = ZerobusStreamOptions { + flush_timeout_ms: 45000, + server_lack_of_ack_timeout_ms: 90000, + }; + + let sdk_options: databricks_zerobus_ingest_sdk::StreamConfigurationOptions = options.into(); + assert_eq!(sdk_options.flush_timeout_ms, 45000); + assert_eq!(sdk_options.server_lack_of_ack_timeout_ms, 90000); + assert_eq!(sdk_options.recovery, true); + assert_eq!(sdk_options.recovery_retries, 4); + } +} diff --git a/src/sinks/databricks_zerobus/error.rs b/src/sinks/databricks_zerobus/error.rs new file mode 100644 index 0000000000000..57598ff3e3435 --- /dev/null +++ b/src/sinks/databricks_zerobus/error.rs @@ -0,0 +1,161 @@ +//! Error types for the Zerobus sink. + +use databricks_zerobus_ingest_sdk::ZerobusError; +use snafu::Snafu; +use vector_lib::event::EventStatus; + +/// Errors that can occur when using the Zerobus sink. +#[derive(Debug, Snafu)] +#[allow(clippy::enum_variant_names)] +pub enum ZerobusSinkError { + /// Configuration validation failed. + #[snafu(display("Configuration error: {}", message))] + ConfigError { message: String }, + + /// Event encoding failed. + #[snafu(display("Encoding error: {}", message))] + EncodingError { message: String }, + + /// Zerobus SDK error. + #[snafu(display("Zerobus error: {}", source))] + ZerobusError { source: ZerobusError }, + + /// Stream initialization failed. + #[snafu(display("Stream initialization failed: {}", source))] + StreamInitError { source: ZerobusError }, + + /// Record ingestion failed. + #[snafu(display("Record ingestion failed: {}", source))] + IngestionError { source: ZerobusError }, +} + +impl From for ZerobusSinkError { + fn from(error: ZerobusError) -> Self { + ZerobusSinkError::ZerobusError { source: error } + } +} + +/// Convert Zerobus errors to Vector event status. +impl From for EventStatus { + fn from(error: ZerobusSinkError) -> Self { + match error { + ZerobusSinkError::ConfigError { .. } | ZerobusSinkError::EncodingError { .. } => { + EventStatus::Rejected + } + ZerobusSinkError::ZerobusError { source } + | ZerobusSinkError::StreamInitError { source } + | ZerobusSinkError::IngestionError { source } => { + if source.is_retryable() { + EventStatus::Errored + } else { + EventStatus::Rejected + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::sinks::databricks_zerobus::service::ZerobusRetryLogic; + use crate::sinks::util::retries::RetryLogic; + + fn retryable_error() -> ZerobusError { + // ChannelCreationError is always retryable + ZerobusError::ChannelCreationError("connection reset".to_string()) + } + + fn non_retryable_error() -> ZerobusError { + // InvalidArgument is never retryable + ZerobusError::InvalidArgument("bad field".to_string()) + } + + #[test] + fn retryable_ingestion_error_maps_to_errored() { + let error = ZerobusSinkError::IngestionError { + source: retryable_error(), + }; + assert_eq!(EventStatus::from(error), EventStatus::Errored); + } + + #[test] + fn non_retryable_ingestion_error_maps_to_rejected() { + let error = ZerobusSinkError::IngestionError { + source: non_retryable_error(), + }; + assert_eq!(EventStatus::from(error), EventStatus::Rejected); + } + + #[test] + fn retryable_stream_init_error_maps_to_errored() { + let error = ZerobusSinkError::StreamInitError { + source: retryable_error(), + }; + assert_eq!(EventStatus::from(error), EventStatus::Errored); + } + + #[test] + fn non_retryable_stream_init_error_maps_to_rejected() { + let error = ZerobusSinkError::StreamInitError { + source: non_retryable_error(), + }; + assert_eq!(EventStatus::from(error), EventStatus::Rejected); + } + + #[test] + fn config_error_maps_to_rejected() { + let error = ZerobusSinkError::ConfigError { + message: "bad config".to_string(), + }; + assert_eq!(EventStatus::from(error), EventStatus::Rejected); + } + + #[test] + fn encoding_error_maps_to_rejected() { + let error = ZerobusSinkError::EncodingError { + message: "encode failed".to_string(), + }; + assert_eq!(EventStatus::from(error), EventStatus::Rejected); + } + + #[test] + fn retry_logic_retryable_errors() { + let logic = ZerobusRetryLogic; + + let error = ZerobusSinkError::IngestionError { + source: retryable_error(), + }; + assert!(logic.is_retriable_error(&error)); + + let error = ZerobusSinkError::StreamInitError { + source: retryable_error(), + }; + assert!(logic.is_retriable_error(&error)); + + let error = ZerobusSinkError::ZerobusError { + source: retryable_error(), + }; + assert!(logic.is_retriable_error(&error)); + } + + #[test] + fn retry_logic_non_retryable_errors() { + let logic = ZerobusRetryLogic; + + let error = ZerobusSinkError::IngestionError { + source: non_retryable_error(), + }; + assert!(!logic.is_retriable_error(&error)); + + let error = ZerobusSinkError::ConfigError { + message: "bad".to_string(), + }; + assert!(!logic.is_retriable_error(&error)); + + let error = ZerobusSinkError::EncodingError { + message: "bad".to_string(), + }; + assert!(!logic.is_retriable_error(&error)); + } +} diff --git a/src/sinks/databricks_zerobus/mod.rs b/src/sinks/databricks_zerobus/mod.rs new file mode 100644 index 0000000000000..08bd685b872d7 --- /dev/null +++ b/src/sinks/databricks_zerobus/mod.rs @@ -0,0 +1,14 @@ +//! The Zerobus sink. +//! +//! This sink streams observability data to Databricks Unity Catalog tables +//! via the Zerobus/Shinkansen ingestion service. + +mod config; +mod error; +#[cfg(feature = "codecs-arrow")] +mod proto_to_arrow; +mod service; +mod sink; +mod unity_catalog_schema; + +pub use config::ZerobusSinkConfig; diff --git a/src/sinks/databricks_zerobus/proto_to_arrow.rs b/src/sinks/databricks_zerobus/proto_to_arrow.rs new file mode 100644 index 0000000000000..6edc54bb2517d --- /dev/null +++ b/src/sinks/databricks_zerobus/proto_to_arrow.rs @@ -0,0 +1,199 @@ +//! Conversion from protobuf `MessageDescriptor` to Arrow `Schema`. +//! +//! Maps protobuf field types to their Arrow equivalents so that the +//! `ArrowStreamSerializer` can encode Vector events using the table's +//! protobuf schema definition. + +use std::sync::Arc; + +use arrow::datatypes::{DataType, Field, Fields, Schema}; +use prost_reflect::{Cardinality, FieldDescriptor, Kind, MessageDescriptor}; + +use super::error::ZerobusSinkError; + +/// Convert a protobuf `MessageDescriptor` into an Arrow `Schema`. +pub fn proto_descriptor_to_arrow_schema( + descriptor: &MessageDescriptor, +) -> Result { + let fields: Result, _> = descriptor + .fields() + .map(|field| proto_field_to_arrow_field(&field)) + .collect(); + + Ok(Schema::new(fields?)) +} + +/// Convert a single protobuf field descriptor into an Arrow `Field`. +fn proto_field_to_arrow_field(field: &FieldDescriptor) -> Result { + let name = field.name().to_string(); + let nullable = field.cardinality() != Cardinality::Required; + + if field.is_map() { + return Err(ZerobusSinkError::ConfigError { + message: format!( + "Map fields are not supported in proto-to-Arrow conversion (field: '{}')", + name + ), + }); + } + + let data_type = proto_kind_to_arrow_type(&field.kind(), &name)?; + + if field.is_list() { + Ok(Field::new( + name, + DataType::List(Arc::new(Field::new("item", data_type, true))), + nullable, + )) + } else { + Ok(Field::new(name, data_type, nullable)) + } +} + +/// Map a protobuf `Kind` to an Arrow `DataType`. +fn proto_kind_to_arrow_type(kind: &Kind, _field_name: &str) -> Result { + match kind { + Kind::Double => Ok(DataType::Float64), + Kind::Float => Ok(DataType::Float32), + Kind::Int32 | Kind::Sint32 | Kind::Sfixed32 => Ok(DataType::Int32), + Kind::Int64 | Kind::Sint64 | Kind::Sfixed64 => Ok(DataType::Int64), + Kind::Uint32 | Kind::Fixed32 => Ok(DataType::UInt32), + Kind::Uint64 | Kind::Fixed64 => Ok(DataType::UInt64), + Kind::Bool => Ok(DataType::Boolean), + Kind::String => Ok(DataType::LargeUtf8), + Kind::Bytes => Ok(DataType::LargeBinary), + Kind::Enum(_) => Ok(DataType::Int32), + Kind::Message(msg_descriptor) => { + let fields: Result, _> = msg_descriptor + .fields() + .map(|f| proto_field_to_arrow_field(&f)) + .collect(); + Ok(DataType::Struct(Fields::from(fields?))) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::Path; + use vrl::protobuf::descriptor::get_message_descriptor; + + /// Load the test User descriptor from the test .desc file. + /// Message: test_proto.User { string id, string name, int32 age, repeated string emails } + fn load_test_user_descriptor() -> MessageDescriptor { + let path = Path::new("tests/data/protobuf/test_proto.desc"); + get_message_descriptor(path, "test_proto.User") + .expect("Failed to load test_proto.User descriptor") + } + + #[test] + fn test_user_schema_fields() { + let descriptor = load_test_user_descriptor(); + let schema = proto_descriptor_to_arrow_schema(&descriptor).unwrap(); + + assert_eq!(schema.fields().len(), 4); + + let id_field = schema.field_with_name("id").unwrap(); + assert_eq!(id_field.data_type(), &DataType::LargeUtf8); + + let name_field = schema.field_with_name("name").unwrap(); + assert_eq!(name_field.data_type(), &DataType::LargeUtf8); + + let age_field = schema.field_with_name("age").unwrap(); + assert_eq!(age_field.data_type(), &DataType::Int32); + } + + #[test] + fn test_repeated_field_becomes_list() { + let descriptor = load_test_user_descriptor(); + let schema = proto_descriptor_to_arrow_schema(&descriptor).unwrap(); + + let emails_field = schema.field_with_name("emails").unwrap(); + match emails_field.data_type() { + DataType::List(inner) => { + assert_eq!(inner.data_type(), &DataType::LargeUtf8); + } + other => panic!("Expected List, got {:?}", other), + } + } + + #[test] + fn test_proto3_fields_are_nullable() { + let descriptor = load_test_user_descriptor(); + let schema = proto_descriptor_to_arrow_schema(&descriptor).unwrap(); + + for field in schema.fields() { + assert!( + field.is_nullable(), + "Field '{}' should be nullable in proto3", + field.name() + ); + } + } + + #[test] + fn test_nested_message_from_unity_catalog() { + use super::super::unity_catalog_schema::{ + UnityCatalogColumn, UnityCatalogTableSchema, generate_descriptor_from_schema, + }; + + let schema = UnityCatalogTableSchema { + name: "test_table".to_string(), + catalog_name: "test_catalog".to_string(), + schema_name: "test_schema".to_string(), + columns: vec![ + UnityCatalogColumn { + name: "id".to_string(), + type_text: "LONG".to_string(), + type_name: "LONG".to_string(), + position: 0, + nullable: false, + type_json: String::new(), + }, + UnityCatalogColumn { + name: "name".to_string(), + type_text: "STRING".to_string(), + type_name: "STRING".to_string(), + position: 1, + nullable: true, + type_json: String::new(), + }, + UnityCatalogColumn { + name: "score".to_string(), + type_text: "DOUBLE".to_string(), + type_name: "DOUBLE".to_string(), + position: 2, + nullable: true, + type_json: String::new(), + }, + UnityCatalogColumn { + name: "active".to_string(), + type_text: "BOOLEAN".to_string(), + type_name: "BOOLEAN".to_string(), + position: 3, + nullable: false, + type_json: String::new(), + }, + ], + }; + + let descriptor = + generate_descriptor_from_schema(&schema).expect("Failed to generate descriptor"); + let arrow_schema = proto_descriptor_to_arrow_schema(&descriptor).unwrap(); + + assert_eq!(arrow_schema.fields().len(), 4); + + let id_field = arrow_schema.field_with_name("id").unwrap(); + assert_eq!(id_field.data_type(), &DataType::Int64); + + let name_field = arrow_schema.field_with_name("name").unwrap(); + assert_eq!(name_field.data_type(), &DataType::LargeUtf8); + + let score_field = arrow_schema.field_with_name("score").unwrap(); + assert_eq!(score_field.data_type(), &DataType::Float64); + + let active_field = arrow_schema.field_with_name("active").unwrap(); + assert_eq!(active_field.data_type(), &DataType::Boolean); + } +} diff --git a/src/sinks/databricks_zerobus/service.rs b/src/sinks/databricks_zerobus/service.rs new file mode 100644 index 0000000000000..5e101e9a9f356 --- /dev/null +++ b/src/sinks/databricks_zerobus/service.rs @@ -0,0 +1,635 @@ +//! Zerobus service wrapper for Vector sink integration. + +#[cfg(feature = "codecs-arrow")] +use databricks_zerobus_ingest_sdk::{ArrowTableProperties, ZerobusArrowStream}; +use databricks_zerobus_ingest_sdk::{TableProperties, ZerobusSdk, ZerobusStream}; +use futures::future::BoxFuture; +use std::path::Path; +use std::sync::Arc; +use tokio::sync::Mutex; +use tower::Service; +use tracing::warn; +use vector_lib::finalization::{EventFinalizers, Finalizable}; +use vector_lib::request_metadata::{GroupedCountByteSize, MetaDescriptive, RequestMetadata}; +use vector_lib::stream::DriverResponse; +use vrl::protobuf::descriptor::get_message_descriptor; + +use crate::sinks::util::retries::RetryLogic; + +use super::{config::ZerobusSinkConfig, error::ZerobusSinkError, unity_catalog_schema}; + +/// The payload for a Zerobus request. +/// +/// Proto payloads contain pre-encoded protobuf records, while Arrow payloads +/// carry a RecordBatch for Arrow Flight ingestion. +#[derive(Clone, Debug)] +pub enum ZerobusPayload { + /// Pre-encoded protobuf records (one byte buffer per event). + Records(Vec>), + /// An Arrow RecordBatch for Arrow Flight ingestion. + #[cfg(feature = "codecs-arrow")] + Arrow(arrow::record_batch::RecordBatch), +} + +/// Request type for the Zerobus service. +#[derive(Clone, Debug)] +pub struct ZerobusRequest { + pub payload: ZerobusPayload, + pub metadata: RequestMetadata, + pub finalizers: EventFinalizers, +} + +/// Response type for the Zerobus service. +#[derive(Debug)] +pub struct ZerobusResponse { + pub events_byte_size: GroupedCountByteSize, +} + +impl DriverResponse for ZerobusResponse { + fn event_status(&self) -> vector_lib::event::EventStatus { + vector_lib::event::EventStatus::Delivered + } + + fn events_sent(&self) -> &GroupedCountByteSize { + &self.events_byte_size + } +} + +impl Finalizable for ZerobusRequest { + fn take_finalizers(&mut self) -> EventFinalizers { + std::mem::take(&mut self.finalizers) + } +} + +impl MetaDescriptive for ZerobusRequest { + fn get_metadata(&self) -> &RequestMetadata { + &self.metadata + } + + fn metadata_mut(&mut self) -> &mut RequestMetadata { + &mut self.metadata + } +} + +/// Determines what kind of stream the service creates and how payloads are ingested. +#[derive(Clone)] +pub enum StreamMode { + /// Proto stream using `ZerobusStream::ingest_records_offset`. + Proto { + descriptor_proto: Arc, + }, + /// Arrow Flight stream using `ZerobusArrowStream::ingest_batch`. + #[cfg(feature = "codecs-arrow")] + Arrow { + arrow_schema: Arc, + }, +} + +/// The active stream, either proto or Arrow Flight. +enum ActiveStream { + Proto(Box), + #[cfg(feature = "codecs-arrow")] + Arrow(ZerobusArrowStream), + /// Test-only variant that returns a pre-configured error on ingest. + #[cfg(test)] + Mock(MockStream), +} + +impl ActiveStream { + /// Gracefully flush and close the underlying SDK stream. + /// + /// Safe to call before the value is dropped — the SDK's own `Drop` + /// implementation is a no-op on already-closed streams. + async fn close(&mut self) { + let result = match self { + ActiveStream::Proto(s) => s.close().await, + #[cfg(feature = "codecs-arrow")] + ActiveStream::Arrow(s) => s.close().await, + #[cfg(test)] + ActiveStream::Mock(m) => { + m.closed.store(true, std::sync::atomic::Ordering::Relaxed); + Ok(()) + } + }; + if let Err(e) = result { + warn!(message = "Failed to close Zerobus stream.", error = %e); + } + } +} + +/// A mock stream that returns a configurable error on the next ingest call. +#[cfg(test)] +pub struct MockStream { + /// When `Some`, the next ingest returns this error; when `None`, ingest succeeds. + next_error: std::sync::Mutex>, + /// Shared flag set to `true` when `ActiveStream::close()` is called. + closed: Arc, +} + +#[cfg(test)] +impl MockStream { + pub fn succeeding() -> Self { + Self { + next_error: std::sync::Mutex::new(None), + closed: Arc::new(std::sync::atomic::AtomicBool::new(false)), + } + } + + pub fn failing(error: databricks_zerobus_ingest_sdk::ZerobusError) -> Self { + Self { + next_error: std::sync::Mutex::new(Some(error)), + closed: Arc::new(std::sync::atomic::AtomicBool::new(false)), + } + } + + /// Returns a shared handle to the closed flag for test assertions. + pub fn closed_flag(&self) -> Arc { + Arc::clone(&self.closed) + } + + /// Set the error that will be returned on the next ingest call. + pub fn set_next_error(&self, error: databricks_zerobus_ingest_sdk::ZerobusError) { + *self.next_error.lock().unwrap() = Some(error); + } + + fn try_ingest(&self) -> Result<(), databricks_zerobus_ingest_sdk::ZerobusError> { + match self.next_error.lock().unwrap().take() { + Some(e) => Err(e), + None => Ok(()), + } + } +} + +/// Service for handling Zerobus requests. +pub struct ZerobusService { + sdk: Arc, + config: Arc, + stream: Arc>>>, + stream_mode: StreamMode, + /// When true, the service waits for server-side acknowledgment after each + /// ingest call. Derived from `AcknowledgementsConfig`. + require_acknowledgements: bool, +} + +impl ZerobusService { + pub async fn new( + config: ZerobusSinkConfig, + stream_mode: StreamMode, + require_acknowledgements: bool, + ) -> Result { + // Validate configuration + config.validate()?; + + // Create SDK instance + let sdk = ZerobusSdk::builder() + .endpoint(&config.ingestion_endpoint) + .unity_catalog_url(&config.unity_catalog_endpoint) + .build() + .map_err(|e| ZerobusSinkError::ConfigError { + message: format!("Failed to create Zerobus SDK: {}", e), + })?; + + Ok(Self { + sdk: Arc::new(sdk), + config: Arc::new(config), + stream: Arc::new(Mutex::new(None)), + stream_mode, + require_acknowledgements, + }) + } + + /// Resolve the protobuf message descriptor from the schema configuration. + pub async fn resolve_descriptor( + config: &ZerobusSinkConfig, + ) -> Result { + match &config.schema { + super::config::SchemaSource::Path { path, message_type } => { + let path = Path::new(path); + get_message_descriptor(path, message_type).map_err(|e| { + ZerobusSinkError::ConfigError { + message: format!("Failed to get message descriptor: {}", e), + } + }) + } + super::config::SchemaSource::UnityCatalog => { + let (client_id, client_secret) = match &config.auth { + super::config::DatabricksAuthentication::OAuth { + client_id, + client_secret, + } => (client_id.inner(), client_secret.inner()), + }; + + let table_schema = unity_catalog_schema::fetch_table_schema( + &config.unity_catalog_endpoint, + &config.table_name, + client_id, + client_secret, + ) + .await?; + + unity_catalog_schema::generate_descriptor_from_schema(&table_schema) + } + } + } + + /// Ensure we have an active stream, creating one if necessary. + /// + /// Also used as the healthcheck: eagerly creating a stream verifies + /// OAuth credentials, endpoint connectivity, and table validity. + pub async fn ensure_stream(&self) -> Result<(), ZerobusSinkError> { + self.get_or_create_stream().await.map(|_| ()) + } + + /// Return an `Arc` handle to the active stream, creating one if needed. + /// + /// The lock is held only while checking/creating the stream; callers can + /// then use the returned `Arc` without holding the lock. + async fn get_or_create_stream(&self) -> Result, ZerobusSinkError> { + let mut stream_guard = self.stream.lock().await; + + if stream_guard.is_none() { + let (client_id, client_secret) = match &self.config.auth { + super::config::DatabricksAuthentication::OAuth { + client_id, + client_secret, + } => ( + client_id.inner().to_string(), + client_secret.inner().to_string(), + ), + }; + + let active_stream = match &self.stream_mode { + StreamMode::Proto { descriptor_proto } => { + let table_properties = TableProperties { + table_name: self.config.table_name.clone(), + descriptor_proto: Some((**descriptor_proto).clone()), + }; + let stream_options = Some(self.config.stream_options.clone().into()); + let stream = self + .sdk + .create_stream(table_properties, client_id, client_secret, stream_options) + .await + .map_err(|e| ZerobusSinkError::StreamInitError { source: e })?; + ActiveStream::Proto(Box::new(stream)) + } + #[cfg(feature = "codecs-arrow")] + StreamMode::Arrow { arrow_schema } => { + let table_properties = ArrowTableProperties { + table_name: self.config.table_name.clone(), + schema: Arc::clone(arrow_schema), + }; + let stream_options = Some(self.config.stream_options.clone().into()); + let stream = self + .sdk + .create_arrow_stream( + table_properties, + client_id, + client_secret, + stream_options, + ) + .await + .map_err(|e| ZerobusSinkError::StreamInitError { source: e })?; + ActiveStream::Arrow(stream) + } + }; + + *stream_guard = Some(Arc::new(active_stream)); + } + + Ok(Arc::clone(stream_guard.as_ref().unwrap())) + } + + /// Gracefully close and remove the active stream. + /// + /// Should be called after all in-flight ingests have completed (e.g., + /// after the driver returns) so that the slot holds the sole `Arc` + /// reference to the stream. + pub async fn close_stream(&self) { + if let Some(stream) = self.stream.lock().await.take() { + match Arc::try_unwrap(stream) { + Ok(mut stream) => stream.close().await, + Err(_) => { + warn!( + message = + "Zerobus stream has outstanding references, skipping graceful close." + ); + } + } + } + } + + /// Ingest a payload (proto records or Arrow batch). + /// + /// Obtains an `Arc` handle to the stream (creating one if needed) and + /// then releases the lock before calling into the SDK so that concurrent + /// ingests are not serialized. + /// + /// On retryable errors the active stream is removed from the slot so that + /// the next attempt (driven by Tower retry) creates a fresh one. + pub async fn ingest( + &self, + payload: ZerobusPayload, + events_byte_size: GroupedCountByteSize, + ) -> Result { + let mut stream = self.get_or_create_stream().await?; + + // Lock is not held here — other tasks can ingest concurrently. + let result = match (payload, stream.as_ref()) { + (ZerobusPayload::Records(records), ActiveStream::Proto(stream)) => { + match stream.ingest_records_offset(records).await { + Ok(Some(offset)) if self.require_acknowledgements => { + stream.wait_for_offset(offset).await.map(|_| ()) + } + Ok(_) => Ok(()), + Err(e) => Err(e), + } + } + #[cfg(feature = "codecs-arrow")] + (ZerobusPayload::Arrow(record_batch), ActiveStream::Arrow(stream)) => { + match stream.ingest_batch(record_batch).await { + Ok(offset) if self.require_acknowledgements => { + stream.wait_for_offset(offset).await.map(|_| ()) + } + Ok(_) => Ok(()), + Err(e) => Err(e), + } + } + #[cfg(test)] + (ZerobusPayload::Records(_), ActiveStream::Mock(mock)) => mock.try_ingest(), + #[allow(unreachable_patterns)] + _ => { + return Err(ZerobusSinkError::ConfigError { + message: "Payload type does not match stream type".to_string(), + }); + } + }; + + match result { + Ok(()) => Ok(ZerobusResponse { events_byte_size }), + Err(e) => { + if e.is_retryable() { + // Remove the stream from the slot so the next retry creates a fresh one, + // then try to close gracefully. Dropping the slot's Arc first means our + // local `stream` may be the sole owner, allowing `Arc::get_mut` to succeed. + self.stream.lock().await.take(); + if let Some(active) = Arc::get_mut(&mut stream) { + active.close().await; + } + } + Err(ZerobusSinkError::IngestionError { source: e }) + } + } + } +} + +impl Service for ZerobusService { + type Response = ZerobusResponse; + type Error = ZerobusSinkError; + type Future = BoxFuture<'static, Result>; + + fn poll_ready( + &mut self, + _cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + std::task::Poll::Ready(Ok(())) + } + + fn call(&mut self, mut request: ZerobusRequest) -> Self::Future { + let service = self.clone(); + let events_byte_size = + std::mem::take(request.metadata_mut()).into_events_estimated_json_encoded_byte_size(); + + Box::pin(async move { service.ingest(request.payload, events_byte_size).await }) + } +} + +impl Clone for ZerobusService { + fn clone(&self) -> Self { + Self { + sdk: Arc::clone(&self.sdk), + config: Arc::clone(&self.config), + stream: Arc::clone(&self.stream), + stream_mode: self.stream_mode.clone(), + require_acknowledgements: self.require_acknowledgements, + } + } +} + +/// Retry logic for the Zerobus service. +/// +/// For SDK errors (`ZerobusError`), delegates to the SDK's `is_retryable()` which +/// correctly marks transient errors (stream closed, channel issues) as retriable +/// and permanent errors (invalid table name, invalid argument, invalid endpoint) +/// as non-retriable. +#[derive(Debug, Default, Clone)] +pub struct ZerobusRetryLogic; + +#[cfg(test)] +impl ZerobusService { + /// Create a service with a mock stream already installed for testing. + pub async fn new_with_mock( + config: ZerobusSinkConfig, + mock: MockStream, + require_acknowledgements: bool, + ) -> Result { + config.validate()?; + + let sdk = ZerobusSdk::builder() + .endpoint(&config.ingestion_endpoint) + .unity_catalog_url(&config.unity_catalog_endpoint) + .build() + .map_err(|e| ZerobusSinkError::ConfigError { + message: format!("Failed to create Zerobus SDK: {}", e), + })?; + + Ok(Self { + sdk: Arc::new(sdk), + config: Arc::new(config), + stream: Arc::new(Mutex::new(Some(Arc::new(ActiveStream::Mock(mock))))), + stream_mode: StreamMode::Proto { + descriptor_proto: Arc::new(Default::default()), + }, + require_acknowledgements, + }) + } + + /// Returns true if the service currently has an active stream. + pub async fn has_active_stream(&self) -> bool { + self.stream.lock().await.is_some() + } +} + +impl RetryLogic for ZerobusRetryLogic { + type Error = ZerobusSinkError; + type Request = ZerobusRequest; + type Response = ZerobusResponse; + + fn is_retriable_error(&self, error: &Self::Error) -> bool { + match error { + ZerobusSinkError::ZerobusError { source } + | ZerobusSinkError::StreamInitError { source } + | ZerobusSinkError::IngestionError { source } => source.is_retryable(), + ZerobusSinkError::ConfigError { .. } | ZerobusSinkError::EncodingError { .. } => false, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::sinks::databricks_zerobus::config::{ + DatabricksAuthentication, SchemaSource, ZerobusStreamOptions, + }; + use databricks_zerobus_ingest_sdk::ZerobusError; + use vector_lib::sensitive_string::SensitiveString; + + fn test_config() -> ZerobusSinkConfig { + ZerobusSinkConfig { + ingestion_endpoint: "https://127.0.0.1:1".to_string(), + table_name: "test.default.logs".to_string(), + unity_catalog_endpoint: "https://127.0.0.1:1".to_string(), + auth: DatabricksAuthentication::OAuth { + client_id: SensitiveString::from("id".to_string()), + client_secret: SensitiveString::from("secret".to_string()), + }, + schema: SchemaSource::Path { + path: "tests/data/protobuf/test_proto.desc".to_string(), + message_type: "test_proto.User".to_string(), + }, + stream_options: ZerobusStreamOptions::default(), + batch_encoding: vector_lib::codecs::encoding::BatchSerializerConfig::ProtoBatch( + Default::default(), + ), + batch: Default::default(), + request: Default::default(), + acknowledgements: Default::default(), + } + } + + fn dummy_payload() -> ZerobusPayload { + ZerobusPayload::Records(vec![vec![1, 2, 3]]) + } + + #[tokio::test] + async fn ingest_succeeds_with_mock_stream() { + let service = ZerobusService::new_with_mock(test_config(), MockStream::succeeding(), false) + .await + .unwrap(); + + let result = service + .ingest(dummy_payload(), GroupedCountByteSize::new_untagged()) + .await; + + assert!(result.is_ok()); + assert!(service.has_active_stream().await); + } + + #[tokio::test] + async fn retryable_error_clears_stream() { + let mock = MockStream::failing(ZerobusError::ChannelCreationError( + "connection reset".to_string(), + )); + let service = ZerobusService::new_with_mock(test_config(), mock, false) + .await + .unwrap(); + + assert!(service.has_active_stream().await); + + let err = service + .ingest(dummy_payload(), GroupedCountByteSize::new_untagged()) + .await + .unwrap_err(); + + assert!(matches!(err, ZerobusSinkError::IngestionError { .. })); + assert!(ZerobusRetryLogic.is_retriable_error(&err)); + // Stream must have been cleared for the next retry. + assert!(!service.has_active_stream().await); + } + + #[tokio::test] + async fn non_retryable_error_keeps_stream() { + let mock = MockStream::failing(ZerobusError::InvalidArgument("bad field".to_string())); + let service = ZerobusService::new_with_mock(test_config(), mock, false) + .await + .unwrap(); + + assert!(service.has_active_stream().await); + + let err = service + .ingest(dummy_payload(), GroupedCountByteSize::new_untagged()) + .await + .unwrap_err(); + + assert!(matches!(err, ZerobusSinkError::IngestionError { .. })); + assert!(!ZerobusRetryLogic.is_retriable_error(&err)); + // Stream should NOT be cleared for non-retryable errors. + assert!(service.has_active_stream().await); + } + + #[tokio::test] + async fn stream_recovers_after_retryable_failure() { + // Simulate: success → retryable failure → success again. + let mock = MockStream::succeeding(); + let service = ZerobusService::new_with_mock(test_config(), mock, false) + .await + .unwrap(); + + // First ingest succeeds. + assert!( + service + .ingest(dummy_payload(), GroupedCountByteSize::new_untagged()) + .await + .is_ok() + ); + assert!(service.has_active_stream().await); + + // Inject a retryable error for the next call. + { + let guard = service.stream.lock().await; + if let Some(arc) = guard.as_ref() { + if let ActiveStream::Mock(mock) = arc.as_ref() { + mock.set_next_error(ZerobusError::ChannelCreationError("reset".to_string())); + } + } + } + + // Second ingest fails and clears the stream. + let err = service + .ingest(dummy_payload(), GroupedCountByteSize::new_untagged()) + .await + .unwrap_err(); + assert!(ZerobusRetryLogic.is_retriable_error(&err)); + assert!(!service.has_active_stream().await); + + // Simulate Tower retry: re-inject a fresh mock stream + // (in production, ensure_stream() would create a new real stream). + *service.stream.lock().await = Some(Arc::new(ActiveStream::Mock(MockStream::succeeding()))); + + // Third ingest succeeds on the new stream. + assert!( + service + .ingest(dummy_payload(), GroupedCountByteSize::new_untagged()) + .await + .is_ok() + ); + assert!(service.has_active_stream().await); + } + + #[tokio::test] + async fn close_stream_calls_close_on_active_stream() { + let mock = MockStream::succeeding(); + let closed = mock.closed_flag(); + + let service = ZerobusService::new_with_mock(test_config(), mock, false) + .await + .unwrap(); + + assert!(service.has_active_stream().await); + assert!(!closed.load(std::sync::atomic::Ordering::Relaxed)); + + service.close_stream().await; + + assert!(!service.has_active_stream().await); + assert!(closed.load(std::sync::atomic::Ordering::Relaxed)); + } +} diff --git a/src/sinks/databricks_zerobus/sink.rs b/src/sinks/databricks_zerobus/sink.rs new file mode 100644 index 0000000000000..762871ebdb4be --- /dev/null +++ b/src/sinks/databricks_zerobus/sink.rs @@ -0,0 +1,125 @@ +//! The main Zerobus sink implementation. + +use std::num::NonZeroUsize; +use std::sync::Arc; + +use futures::stream::BoxStream; + +use vector_lib::codecs::encoding::{BatchEncoder, BatchOutput}; +use vector_lib::event::EventStatus; +use vector_lib::finalization::Finalizable; + +use crate::sinks::prelude::*; +use crate::sinks::util::metadata::RequestMetadataBuilder; +use crate::sinks::util::request_builder::default_request_builder_concurrency_limit; +use crate::sinks::util::{RealtimeSizeBasedDefaultBatchSettings, TowerRequestSettings}; + +use super::service::{ZerobusPayload, ZerobusRequest, ZerobusRetryLogic, ZerobusService}; + +/// The main Zerobus sink. +pub struct ZerobusSink { + service: ZerobusService, + request_limits: TowerRequestSettings, + batch_settings: BatcherSettings, + encoder: BatchEncoder, +} + +impl ZerobusSink { + pub fn new( + service: ZerobusService, + request_limits: TowerRequestSettings, + batch_config: BatchConfig, + encoder: BatchEncoder, + ) -> Result { + let batch_settings = batch_config.into_batcher_settings()?; + + Ok(Self { + service, + request_limits, + batch_settings, + encoder, + }) + } + + fn encode_batch( + encoder: &BatchEncoder, + mut events: Vec, + ) -> Result { + let finalizers = events.take_finalizers(); + let metadata_builder = RequestMetadataBuilder::from_events(&events); + + let batch_output = match encoder.encode_batch(&events) { + Ok(output) => output, + Err(e) => { + finalizers.update_status(EventStatus::Rejected); + return Err(format!("Failed to encode batch: {}", e)); + } + }; + + let (payload, byte_size) = match batch_output { + BatchOutput::Records(records) => { + let size = records.iter().map(|r| r.len()).sum::(); + (ZerobusPayload::Records(records), size) + } + #[cfg(feature = "codecs-arrow")] + BatchOutput::Arrow(record_batch) => { + let size = record_batch.get_array_memory_size(); + (ZerobusPayload::Arrow(record_batch), size) + } + #[allow(unreachable_patterns)] + _ => { + finalizers.update_status(EventStatus::Rejected); + return Err("Unexpected batch output type".to_string()); + } + }; + + let request_size = NonZeroUsize::new(byte_size).unwrap_or(NonZeroUsize::MIN); + let metadata = metadata_builder.with_request_size(request_size); + + Ok(ZerobusRequest { + payload, + metadata, + finalizers, + }) + } + + async fn run_inner(self: Box, input: BoxStream<'_, Event>) -> Result<(), ()> { + let encoder = Arc::new(self.encoder.clone()); + + let result = { + let tower_service = ServiceBuilder::new() + .settings(self.request_limits, ZerobusRetryLogic) + .service(self.service.clone()); + + input + .batched(self.batch_settings.as_byte_size_config()) + .concurrent_map(default_request_builder_concurrency_limit(), move |events| { + let encoder = Arc::clone(&encoder); + Box::pin(async move { Self::encode_batch(&encoder, events) }) + }) + .filter_map(|result| async move { + match result { + Err(error) => { + emit!(SinkRequestBuildError { error }); + None + } + Ok(req) => Some(req), + } + }) + .into_driver(tower_service) + .run() + .await + }; + + self.service.close_stream().await; + + result + } +} + +#[async_trait::async_trait] +impl StreamSink for ZerobusSink { + async fn run(self: Box, input: BoxStream<'_, Event>) -> Result<(), ()> { + self.run_inner(input).await + } +} diff --git a/src/sinks/databricks_zerobus/tests/fixtures/all_primitive_types_schema.json b/src/sinks/databricks_zerobus/tests/fixtures/all_primitive_types_schema.json new file mode 100644 index 0000000000000..d115598f0232d --- /dev/null +++ b/src/sinks/databricks_zerobus/tests/fixtures/all_primitive_types_schema.json @@ -0,0 +1,81 @@ +{ + "name": "all_types_table", + "catalog_name": "test_catalog", + "schema_name": "test_schema", + "table_type": "MANAGED", + "data_source_format": "DELTA", + "columns": [ + { + "name": "col_string", + "type_text": "string", + "type_name": "STRING", + "position": 1, + "nullable": true, + "type_json": "{\"name\":\"col_string\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}" + }, + { + "name": "col_int", + "type_text": "int", + "type_name": "INT", + "position": 2, + "nullable": true, + "type_json": "{\"name\":\"col_int\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}" + }, + { + "name": "col_long", + "type_text": "bigint", + "type_name": "LONG", + "position": 3, + "nullable": true, + "type_json": "{\"name\":\"col_long\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}" + }, + { + "name": "col_double", + "type_text": "double", + "type_name": "DOUBLE", + "position": 4, + "nullable": true, + "type_json": "{\"name\":\"col_double\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}}" + }, + { + "name": "col_float", + "type_text": "float", + "type_name": "FLOAT", + "position": 5, + "nullable": true, + "type_json": "{\"name\":\"col_float\",\"type\":\"float\",\"nullable\":true,\"metadata\":{}}" + }, + { + "name": "col_boolean", + "type_text": "boolean", + "type_name": "BOOLEAN", + "position": 6, + "nullable": true, + "type_json": "{\"name\":\"col_boolean\",\"type\":\"boolean\",\"nullable\":true,\"metadata\":{}}" + }, + { + "name": "col_binary", + "type_text": "binary", + "type_name": "BINARY", + "position": 7, + "nullable": true, + "type_json": "{\"name\":\"col_binary\",\"type\":\"binary\",\"nullable\":true,\"metadata\":{}}" + }, + { + "name": "col_timestamp", + "type_text": "timestamp", + "type_name": "TIMESTAMP", + "position": 8, + "nullable": true, + "type_json": "{\"name\":\"col_timestamp\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}}" + }, + { + "name": "col_date", + "type_text": "date", + "type_name": "DATE", + "position": 9, + "nullable": true, + "type_json": "{\"name\":\"col_date\",\"type\":\"date\",\"nullable\":true,\"metadata\":{}}" + } + ] +} diff --git a/src/sinks/databricks_zerobus/tests/fixtures/array_of_structs_schema.json b/src/sinks/databricks_zerobus/tests/fixtures/array_of_structs_schema.json new file mode 100644 index 0000000000000..f58ae4f8ff220 --- /dev/null +++ b/src/sinks/databricks_zerobus/tests/fixtures/array_of_structs_schema.json @@ -0,0 +1,19 @@ +{ + "name": "array_of_structs_table", + "catalog_name": "test_catalog", + "schema_name": "test_schema", + "table_type": "MANAGED", + "data_source_format": "DELTA", + "columns": [ + { + "name": "transactions", + "type_text": "array>", + "type_name": "ARRAY", + "position": 1, + "type_precision": 0, + "type_scale": 0, + "nullable": true, + "type_json": "{\"name\":\"transactions\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"amount\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}},{\"name\":\"currency\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]},\"containsNull\":true},\"nullable\":true,\"metadata\":{}}" + } + ] +} diff --git a/src/sinks/databricks_zerobus/tests/fixtures/complex_nested_types_schema.json b/src/sinks/databricks_zerobus/tests/fixtures/complex_nested_types_schema.json new file mode 100644 index 0000000000000..c351662578c0c --- /dev/null +++ b/src/sinks/databricks_zerobus/tests/fixtures/complex_nested_types_schema.json @@ -0,0 +1,735 @@ +{ + "name": "complex_test_table", + "catalog_name": "test_catalog", + "schema_name": "test_schema", + "columns": [ + { + "name": "field_001", + "type_text": "timestamp", + "type_name": "TIMESTAMP", + "position": 0, + "type_json": "{\"name\": \"field_001\", \"type\": \"timestamp\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_002", + "type_text": "string", + "type_name": "STRING", + "position": 1, + "type_json": "{\"name\": \"field_002\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_003", + "type_text": "int", + "type_name": "INT", + "position": 2, + "type_json": "{\"name\": \"field_003\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_004", + "type_text": "string", + "type_name": "STRING", + "position": 3, + "type_json": "{\"name\": \"field_004\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_005", + "type_text": "string", + "type_name": "STRING", + "position": 4, + "type_json": "{\"name\": \"field_005\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_006", + "type_text": "string", + "type_name": "STRING", + "position": 5, + "type_json": "{\"name\": \"field_006\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_007", + "type_text": "struct,field_030:string,field_031:bigint,field_032:bigint,field_033:string,field_034:string,field_035:string,field_036:string,field_037:struct,field_044:string,field_045:string,field_046:string>,field_047:struct,field_069:struct,field_084:struct>,field_087:struct>>", + "type_name": "STRUCT", + "position": 6, + "type_json": "{\"name\": \"field_007\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_008\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_009\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_010\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_011\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_012\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_013\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_014\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_015\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_016\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_017\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_018\", \"type\": \"binary\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_019\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_020\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_021\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_022\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_023\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_024\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_025\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_026\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_027\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_028\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_029\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_030\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_031\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_032\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_033\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_034\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_035\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_036\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_037\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_038\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_039\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_040\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_041\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_042\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_043\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_044\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_045\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_046\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_047\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_048\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_049\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_050\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_051\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_052\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_053\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_054\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_055\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_056\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_057\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_058\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_059\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_060\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_061\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_062\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_063\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_064\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_065\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_066\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_067\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_068\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_069\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_012\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_070\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_071\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_072\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_073\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_074\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_075\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_032\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_076\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_077\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_078\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_079\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_080\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_024\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_081\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_082\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_083\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_043\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_084\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_085\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_086\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_037\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_038\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_039\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_040\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_041\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_042\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_043\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_087\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_088\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_089\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_037\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_038\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_039\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_040\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_041\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_042\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_043\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_090", + "type_text": "struct", + "type_name": "STRUCT", + "position": 7, + "type_json": "{\"name\": \"field_090\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_091\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_092\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_093\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_094\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_095", + "type_text": "string", + "type_name": "STRING", + "position": 8, + "type_json": "{\"name\": \"field_095\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_096", + "type_text": "string", + "type_name": "STRING", + "position": 9, + "type_json": "{\"name\": \"field_096\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_097", + "type_text": "string", + "type_name": "STRING", + "position": 10, + "type_json": "{\"name\": \"field_097\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_098", + "type_text": "string", + "type_name": "STRING", + "position": 11, + "type_json": "{\"name\": \"field_098\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_099", + "type_text": "string", + "type_name": "STRING", + "position": 12, + "type_json": "{\"name\": \"field_099\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_100", + "type_text": "bigint", + "type_name": "LONG", + "position": 13, + "type_json": "{\"name\": \"field_100\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_101", + "type_text": "bigint", + "type_name": "LONG", + "position": 14, + "type_json": "{\"name\": \"field_101\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_102", + "type_text": "boolean", + "type_name": "BOOLEAN", + "position": 15, + "type_json": "{\"name\": \"field_102\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_103", + "type_text": "boolean", + "type_name": "BOOLEAN", + "position": 16, + "type_json": "{\"name\": \"field_103\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_104", + "type_text": "struct>>", + "type_name": "STRUCT", + "position": 17, + "type_json": "{\"name\": \"field_104\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_105\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_106\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_107\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_108\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_109\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_110\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_105\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_106\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_111", + "type_text": "array", + "type_name": "ARRAY", + "position": 18, + "type_json": "{\"name\": \"field_111\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_112", + "type_text": "struct", + "type_name": "STRUCT", + "position": 19, + "type_json": "{\"name\": \"field_112\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_113\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_114\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_115\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_116\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_117", + "type_text": "struct>,field_131:array>,field_133:array>,field_134:array>,field_139:array>>>,field_142:map,field_143:map,field_144:bigint,field_145:bigint,field_146:bigint,field_147:bigint,field_148:bigint>", + "type_name": "STRUCT", + "position": 20, + "type_json": "{\"name\": \"field_117\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_118\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_119\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_120\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_121\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_122\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_123\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_124\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_125\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_126\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_127\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_128\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_129\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_130\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_131\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_132\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_133\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_132\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_134\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_135\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_136\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_137\", \"type\": \"double\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_138\", \"type\": \"double\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_139\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_135\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_136\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_140\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_141\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_136\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_142\", \"type\": {\"type\": \"map\", \"keyType\": \"long\", \"valueType\": \"boolean\", \"valueContainsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_143\", \"type\": {\"type\": \"map\", \"keyType\": \"long\", \"valueType\": \"double\", \"valueContainsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_144\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_145\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_146\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_147\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_148\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_149", + "type_text": "array,field_177:struct,field_180:struct,field_188:struct,field_190:struct,field_195:bigint,field_196:bigint,field_197:bigint,field_198:bigint,field_199:bigint,field_200:bigint,field_201:bigint,field_202:bigint,field_203:bigint,field_204:bigint,field_205:bigint>,field_206:struct,field_177:struct,field_180:struct,field_188:struct,field_190:struct,field_195:bigint,field_196:bigint,field_197:bigint,field_198:bigint,field_199:bigint,field_200:bigint,field_201:bigint,field_202:bigint,field_203:bigint,field_204:bigint,field_205:bigint>,field_207:string,field_208:bigint,field_209:bigint,field_210:bigint,field_211:bigint,field_109:string,field_212:boolean,field_213:string,field_214:string,field_215:bigint,field_216:bigint,field_217:bigint,field_218:bigint,field_219:boolean,field_220:int,field_221:int,field_222:array>,field_240:bigint,field_241:int,field_242:bigint,field_243:bigint,field_244:array>,field_248:string,field_249:string,field_250:string,field_251:string,field_252:array,field_162:string,field_109:string>>,field_254:struct,field_261:struct,field_262:boolean>>", + "type_name": "ARRAY", + "position": 21, + "type_json": "{\"name\": \"field_149\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_150\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_151\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_152\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_153\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_154\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_155\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_156\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_157\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_158\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_159\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_160\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_161\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_162\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_163\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_164\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_165\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_166\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_167\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_168\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_169\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_170\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_171\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_172\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_173\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_174\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_175\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_176\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_177\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_178\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_179\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_180\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_181\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_182\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_183\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_184\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_185\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_186\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_176\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_175\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_187\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_188\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_178\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_189\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_179\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_190\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_191\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_192\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_193\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_194\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_195\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_196\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_197\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_198\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_199\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_200\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_201\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_202\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_203\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_204\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_205\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_206\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_164\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_165\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_166\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_167\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_168\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_169\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_170\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_171\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_172\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_173\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_174\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_175\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_176\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_177\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_178\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_179\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_180\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_181\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_182\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_183\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_184\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_185\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_186\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_176\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_175\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_187\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_188\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_178\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_189\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_179\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_190\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_191\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_192\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_193\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_194\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_195\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_196\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_197\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_198\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_199\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_200\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_201\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_202\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_203\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_204\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_205\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_207\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_208\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_209\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_210\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_211\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_109\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_212\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_213\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_214\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_215\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_216\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_217\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_218\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_219\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_220\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_221\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_222\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_223\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_224\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_225\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_226\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_227\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_228\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_229\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_230\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_231\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_232\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_233\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_234\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_235\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_236\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_237\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_238\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_239\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_240\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_241\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_242\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_243\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_244\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_245\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_246\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_247\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_248\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_249\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_250\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_251\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_252\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_253\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_223\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_224\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_225\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_226\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_227\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_228\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_229\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_230\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_231\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_232\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_233\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_234\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_235\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_236\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_237\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_238\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_239\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_162\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_109\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_254\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_255\", \"type\": \"double\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_256\", \"type\": \"double\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_257\", \"type\": \"double\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_258\", \"type\": \"double\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_259\", \"type\": \"double\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_260\", \"type\": \"double\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_261\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_255\", \"type\": \"double\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_256\", \"type\": \"double\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_257\", \"type\": \"double\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_258\", \"type\": \"double\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_259\", \"type\": \"double\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_260\", \"type\": \"double\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_262\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_263", + "type_text": "string", + "type_name": "STRING", + "position": 22, + "type_json": "{\"name\": \"field_263\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_264", + "type_text": "struct", + "type_name": "STRUCT", + "position": 23, + "type_json": "{\"name\": \"field_264\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_265\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_266\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_267\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_268\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_269\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_270\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_271\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_272\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_273\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_274\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_275\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_276\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_277\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_278", + "type_text": "array>,field_283:array>,field_284:array,field_285:array,field_286:struct>>,field_293:struct,field_295:struct,field_298:struct>,field_299:struct,field_300:struct>,field_303:struct,field_294:bigint>,field_304:struct>>,field_305:boolean,field_302:bigint,field_306:array>,field_307:struct>,field_308:struct>>,field_310:string>,field_311:struct>>>,field_312:struct,field_317:bigint>,field_318:struct,field_321:array,field_323:struct,field_324:array,field_141:string,field_326:string>>>>,field_327:struct,field_330:struct,field_335:array,field_336:string,field_337:string>>,field_338:array>,field_284:array,field_339:int,field_340:struct>,field_346:string>>,field_347:struct>>,field_305:boolean,field_302:bigint,field_306:array>,field_353:struct>>,field_359:string,field_360:string,field_361:bigint,field_362:boolean>,field_363:array>>,field_305:boolean,field_302:bigint,field_306:array>,field_353:struct>>,field_359:string,field_360:string,field_361:bigint,field_362:boolean>,field_364:boolean>>,field_365:struct>>,field_305:boolean,field_302:bigint,field_306:array>,field_353:struct>>,field_359:string,field_360:string,field_361:bigint,field_362:boolean>,field_366:int,field_367:array>>,field_305:boolean,field_302:bigint,field_306:array>,field_353:struct>>,field_359:string,field_360:string,field_361:bigint,field_362:boolean>,field_364:boolean>>,field_368:struct>>,field_305:boolean,field_302:bigint,field_306:array>,field_353:struct>>,field_359:string,field_360:string,field_361:bigint,field_362:boolean>,field_369:struct,field_375:struct>>,field_305:boolean,field_302:bigint,field_306:array>,field_353:struct>>,field_359:string,field_360:string,field_361:bigint,field_362:boolean>,field_376:struct>>,field_305:boolean,field_302:bigint,field_306:array>,field_353:struct>>,field_359:string,field_360:string,field_361:bigint,field_362:boolean>,field_335:array>,field_377:array,field_378:boolean,field_379:struct,field_392:string>,field_393:struct>>,field_348:bigint,field_398:bigint,field_399:struct>>>,field_402:array,field_207:string>,field_397:struct>>>,field_403:boolean,field_404:boolean,field_405:array,field_406:struct>>,field_407:boolean,field_408:struct>,field_410:array,field_411:array>>,field_412:struct,field_420:string,field_421:struct,field_428:struct>,field_431:array>,field_440:struct>,field_442:array>,field_443:array>,field_444:array>,field_445:array>,field_446:array>,field_447:array>,field_448:array>>,field_449:struct>>,field_305:boolean,field_302:bigint,field_306:array>,field_460:string>>>,field_461:boolean,field_462:boolean>,field_463:struct,field_468:struct,field_471:struct,field_474:array,field_475:array>,field_476:struct,field_474:array,field_475:array>,field_478:struct>,field_480:struct,field_482:struct,field_483:struct,field_485:struct,field_488:array>,field_489:struct,field_488:array>,field_490:struct,field_495:array,field_496:array,field_497:array,field_498:array>,field_499:struct>,field_500:struct,field_503:struct,field_506:array,field_507:int,field_508:int,field_509:string,field_510:boolean,field_511:boolean,field_512:boolean,field_513:int,field_514:array,field_515:array,field_516:bigint,field_517:array>,field_521:array>>>>>,field_525:struct>,field_410:array,field_411:array>>,field_365:struct>>,field_305:boolean,field_302:bigint,field_306:array>,field_353:struct>>,field_359:string,field_360:string,field_361:bigint,field_362:boolean>,field_526:struct>>,field_305:boolean,field_302:bigint,field_306:array>,field_353:struct>>,field_359:string,field_360:string,field_361:bigint,field_362:boolean>,field_527:struct>>,field_305:boolean,field_302:bigint,field_306:array>,field_353:struct>>,field_359:string,field_360:string,field_361:bigint,field_362:boolean>,field_528:array,field_534:struct,field_535:double,field_536:double,field_537:boolean>>>>>>>,field_538:struct,field_544:struct,field_546:struct>,field_549:string,field_525:struct>,field_410:array,field_411:array>>>,field_550:struct,field_553:struct,field_554:struct,field_392:string>,field_555:struct,field_392:string>,field_314:string,field_556:struct>,field_558:string,field_559:boolean,field_557:array>,field_560:struct>,field_562:struct>,field_563:struct>,field_501:bigint,field_566:array>>,field_568:array>>,field_569:struct>>,field_570:array>>,field_380:struct,field_392:string>,field_571:boolean>,field_572:struct,field_392:string>,field_573:string,field_574:array>,field_575:struct,field_576:struct,field_577:struct,field_578:struct>,field_580:struct>,field_581:struct>,field_582:struct>>,field_557:array,field_590:struct,field_323:struct,field_324:array,field_141:string,field_326:string>>>,field_591:struct>,field_593:bigint,field_594:struct,field_596:array,field_393:struct,field_597:array,field_598:int>>,field_599:boolean,field_600:struct,field_607:struct,field_609:struct,field_611:struct,field_612:struct,field_613:struct,field_614:struct,field_615:boolean,field_616:struct,field_618:struct,field_619:struct>,field_410:array,field_411:array>>>,field_334:boolean,field_620:boolean,field_621:struct,field_625:array>,field_626:struct,field_630:struct,field_633:array>,field_634:string,field_574:array,field_314:string,field_380:struct,field_392:string>,field_396:struct,field_397:struct>>,field_348:bigint,field_398:bigint,field_399:struct>>>>>>>,field_635:struct,field_382:struct>>>,field_638:struct,field_392:string>,field_639:string>>,field_301:struct>>,field_640:struct,field_022:string,field_653:boolean,field_654:boolean,field_655:bigint,field_656:bigint,field_657:boolean,field_658:boolean,field_659:boolean,field_660:boolean,field_661:boolean,field_662:boolean,field_663:boolean,field_664:boolean,field_665:boolean,field_666:string,field_667:int>>,field_668:struct,field_669:struct,field_670:struct,field_671:struct,field_672:struct>,field_675:boolean>,field_676:struct,field_678:struct,field_679:struct,field_680:struct,field_681:struct,field_682:struct,field_683:struct>>", + "type_name": "ARRAY", + "position": 24, + "type_json": "{\"name\": \"field_278\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_279\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_095\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_280\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_281\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_282\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_095\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_279\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_283\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_279\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_095\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_280\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_281\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_282\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_095\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_279\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_284\", \"type\": {\"type\": \"array\", \"elementType\": \"long\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_285\", \"type\": {\"type\": \"array\", \"elementType\": \"long\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_286\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_287\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_288\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_289\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_290\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_291\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_292\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_293\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_294\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_295\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_296\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_297\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_298\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_282\", \"type\": {\"type\": \"array\", \"elementType\": \"long\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_299\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_294\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_300\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_301\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_302\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_095\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_303\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_301\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_302\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_095\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_294\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_304\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_207\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_286\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_287\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_288\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_289\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_290\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_291\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_292\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_305\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_302\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_306\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_307\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_207\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_302\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_306\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_308\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_294\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_309\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_287\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_288\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_289\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_290\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_291\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_292\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_310\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_311\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_286\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_287\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_288\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_289\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_290\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_291\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_292\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_312\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_313\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_314\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_315\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_316\", \"type\": {\"type\": \"array\", \"elementType\": \"long\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_317\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_318\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_294\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_319\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_320\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_321\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_322\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_207\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_323\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_207\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_324\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_325\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_207\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_141\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_326\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_327\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_141\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_328\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_329\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_330\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_331\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_332\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_333\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_334\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_335\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_336\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_337\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_338\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_095\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_279\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_284\", \"type\": {\"type\": \"array\", \"elementType\": \"long\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_339\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_340\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_341\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_207\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_342\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_343\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_344\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_345\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_141\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_346\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_347\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_348\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_349\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_350\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_351\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_352\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_207\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_286\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_287\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_288\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_289\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_290\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_291\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_292\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_305\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_302\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_306\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_353\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_354\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_355\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_356\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_357\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_358\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_359\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_360\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_361\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_362\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_363\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_095\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_347\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_348\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_349\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_350\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_351\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_352\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_207\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_286\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_287\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_288\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_289\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_290\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_291\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_292\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_305\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_302\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_306\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_353\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_354\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_355\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_356\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_357\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_358\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_359\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_360\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_361\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_362\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_364\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_365\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_348\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_349\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_350\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_351\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_352\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_207\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_286\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_287\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_288\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_289\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_290\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_291\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_292\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_305\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_302\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_306\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_353\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_354\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_355\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_356\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_357\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_358\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_359\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_360\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_361\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_362\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_366\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_367\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_095\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_347\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_348\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_349\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_350\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_351\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_352\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_207\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_286\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_287\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_288\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_289\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_290\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_291\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_292\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_305\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_302\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_306\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_353\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_354\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_355\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_356\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_357\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_358\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_359\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_360\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_361\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_362\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_364\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_368\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_348\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_349\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_350\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_351\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_352\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_207\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_286\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_287\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_288\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_289\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_290\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_291\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_292\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_305\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_302\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_306\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_353\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_354\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_355\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_356\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_357\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_358\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_359\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_360\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_361\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_362\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_369\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_370\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_371\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_372\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_373\", \"type\": \"double\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_374\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_375\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_348\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_349\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_350\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_351\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_352\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_207\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_286\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_287\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_288\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_289\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_290\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_291\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_292\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_305\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_302\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_306\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_353\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_354\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_355\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_356\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_357\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_358\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_359\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_360\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_361\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_362\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_376\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_348\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_349\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_350\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_351\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_352\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_207\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_286\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_287\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_288\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_289\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_290\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_291\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_292\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_305\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_302\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_306\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_353\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_354\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_355\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_356\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_357\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_358\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_359\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_360\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_361\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_362\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_335\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_377\", \"type\": {\"type\": \"array\", \"elementType\": \"integer\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_378\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_379\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_380\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_381\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_382\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_383\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_384\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_385\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_386\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_387\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_388\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_389\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_390\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_391\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_392\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_393\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_207\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_394\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_395\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_396\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_397\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_287\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_288\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_289\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_290\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_291\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_292\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_348\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_398\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_399\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_400\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_401\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_402\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_207\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_397\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_287\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_288\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_289\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_290\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_291\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_292\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_403\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_404\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_405\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_406\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_287\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_288\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_289\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_290\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_291\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_292\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_407\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_408\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_409\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_105\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_281\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_410\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_411\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_323\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_136\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_412\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_413\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_414\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_415\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_416\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_417\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_418\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_419\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_420\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_421\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_422\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_423\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_424\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_425\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_426\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_427\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_428\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_429\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_430\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_431\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_432\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_433\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_434\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_435\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_436\", \"type\": \"double\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_437\", \"type\": \"double\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_438\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_439\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_440\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_441\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_095\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_279\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_442\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_095\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_279\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_443\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_095\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_279\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_444\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_095\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_279\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_445\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_095\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_279\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_446\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_095\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_279\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_447\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_095\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_279\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_448\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_095\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_279\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_449\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_348\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_450\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_349\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_451\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_452\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_453\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_454\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_455\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_456\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_457\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_458\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_459\", \"type\": \"double\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_352\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_207\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_286\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_287\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_288\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_289\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_290\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_291\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_292\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_305\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_302\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_306\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_460\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_461\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_462\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_463\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_464\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_465\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_466\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_467\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_102\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_468\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_469\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_470\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_471\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_472\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_473\", \"type\": {\"type\": \"array\", \"elementType\": \"long\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_474\", \"type\": {\"type\": \"array\", \"elementType\": \"long\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_475\", \"type\": {\"type\": \"array\", \"elementType\": \"long\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_476\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_477\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_473\", \"type\": {\"type\": \"array\", \"elementType\": \"long\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_474\", \"type\": {\"type\": \"array\", \"elementType\": \"long\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_475\", \"type\": {\"type\": \"array\", \"elementType\": \"long\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_478\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_479\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_318\", \"type\": {\"type\": \"array\", \"elementType\": \"long\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_480\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_481\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_482\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_472\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_483\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_207\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_484\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_485\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_207\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_486\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_487\", \"type\": {\"type\": \"array\", \"elementType\": \"long\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_488\", \"type\": {\"type\": \"array\", \"elementType\": \"integer\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_489\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_207\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_486\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_487\", \"type\": {\"type\": \"array\", \"elementType\": \"long\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_488\", \"type\": {\"type\": \"array\", \"elementType\": \"integer\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_490\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_491\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_492\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_472\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_102\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_493\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_494\", \"type\": {\"type\": \"array\", \"elementType\": \"long\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_495\", \"type\": {\"type\": \"array\", \"elementType\": \"long\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_496\", \"type\": {\"type\": \"array\", \"elementType\": \"long\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_497\", \"type\": {\"type\": \"array\", \"elementType\": \"long\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_498\", \"type\": {\"type\": \"array\", \"elementType\": \"long\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_499\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_473\", \"type\": {\"type\": \"array\", \"elementType\": \"long\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_500\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_501\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_502\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_503\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_504\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_501\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_505\", \"type\": {\"type\": \"array\", \"elementType\": \"long\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_506\", \"type\": {\"type\": \"array\", \"elementType\": \"long\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_507\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_508\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_509\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_510\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_511\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_512\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_513\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_514\", \"type\": {\"type\": \"array\", \"elementType\": \"integer\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_515\", \"type\": {\"type\": \"array\", \"elementType\": \"integer\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_516\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_517\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_518\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_519\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_520\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_136\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_521\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_522\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_523\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_509\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_524\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_520\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_136\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_525\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_409\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_105\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_281\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_410\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_411\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_323\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_136\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_365\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_348\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_349\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_350\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_351\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_352\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_207\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_286\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_287\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_288\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_289\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_290\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_291\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_292\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_305\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_302\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_306\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_353\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_354\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_355\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_356\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_357\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_358\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_359\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_360\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_361\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_362\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_526\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_348\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_349\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_350\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_351\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_352\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_207\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_286\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_287\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_288\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_289\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_290\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_291\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_292\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_305\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_302\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_306\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_353\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_354\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_355\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_356\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_357\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_358\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_359\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_360\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_361\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_362\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_527\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_348\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_349\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_350\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_351\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_352\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_207\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_286\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_287\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_288\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_289\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_290\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_291\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_292\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_305\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_302\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_306\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_353\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_354\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_355\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_356\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_357\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_358\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_359\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_360\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_361\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_362\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_528\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_518\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_529\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_522\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_523\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_509\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_530\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_531\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_349\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_532\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_348\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_533\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_459\", \"type\": \"double\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_534\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_349\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_532\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_348\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_533\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_459\", \"type\": \"double\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_535\", \"type\": \"double\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_536\", \"type\": \"double\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_537\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_538\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_539\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_540\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_541\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_542\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_543\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_544\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_314\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_545\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_546\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_547\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_279\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_548\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_338\", \"type\": {\"type\": \"array\", \"elementType\": \"long\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_549\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_525\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_409\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_105\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_281\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_410\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_411\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_323\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_136\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_550\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_551\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_279\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_552\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_207\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_553\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_135\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_141\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_554\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_382\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_381\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_382\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_383\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_384\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_385\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_386\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_387\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_388\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_389\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_390\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_391\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_392\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_555\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_381\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_382\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_383\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_384\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_385\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_386\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_387\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_388\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_389\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_390\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_391\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_392\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_314\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_556\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_207\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_557\", \"type\": {\"type\": \"array\", \"elementType\": \"long\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_558\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_559\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_557\", \"type\": {\"type\": \"array\", \"elementType\": \"long\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_560\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_561\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_207\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_557\", \"type\": {\"type\": \"array\", \"elementType\": \"long\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_562\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_207\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_557\", \"type\": {\"type\": \"array\", \"elementType\": \"long\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_563\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_279\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_564\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_207\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_565\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_501\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_566\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_279\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_501\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_567\", \"type\": {\"type\": \"array\", \"elementType\": \"long\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_568\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_279\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_501\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_567\", \"type\": {\"type\": \"array\", \"elementType\": \"long\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_569\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_287\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_288\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_289\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_290\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_291\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_292\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_570\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_279\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_501\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_567\", \"type\": {\"type\": \"array\", \"elementType\": \"long\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_380\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_381\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_382\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_383\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_384\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_385\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_386\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_387\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_388\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_389\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_390\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_391\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_392\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_571\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_572\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_380\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_381\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_382\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_383\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_384\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_385\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_386\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_387\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_388\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_389\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_390\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_391\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_392\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_573\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_574\", \"type\": {\"type\": \"array\", \"elementType\": \"long\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_575\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_565\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_576\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_565\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_577\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_565\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_578\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_579\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_580\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_579\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_581\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_579\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_582\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_583\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_584\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_585\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_586\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_587\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_588\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_589\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_390\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_391\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_557\", \"type\": {\"type\": \"array\", \"elementType\": \"long\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_590\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_322\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_207\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_323\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_207\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_324\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_325\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_207\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_141\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_326\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_591\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_207\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_592\", \"type\": {\"type\": \"array\", \"elementType\": \"long\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_593\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_594\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_595\", \"type\": {\"type\": \"array\", \"elementType\": \"long\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_596\", \"type\": {\"type\": \"array\", \"elementType\": \"long\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_393\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_557\", \"type\": {\"type\": \"array\", \"elementType\": \"long\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_597\", \"type\": {\"type\": \"array\", \"elementType\": \"long\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_598\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_599\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_600\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_601\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_602\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_021\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_035\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_603\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_604\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_605\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_606\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_607\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_608\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_609\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_207\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_610\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_611\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_608\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_612\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_207\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_613\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_207\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_614\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_207\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_615\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_616\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_420\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_617\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_618\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_420\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_617\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_619\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_525\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_409\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_105\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_281\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_410\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_411\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_323\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_136\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_334\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_620\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_621\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_622\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_279\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_623\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_280\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_624\", \"type\": {\"type\": \"array\", \"elementType\": \"long\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_625\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_626\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_627\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_628\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_629\", \"type\": {\"type\": \"array\", \"elementType\": \"long\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_630\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_631\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_632\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_633\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_634\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_574\", \"type\": {\"type\": \"array\", \"elementType\": \"long\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_314\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_380\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_381\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_382\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_383\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_384\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_385\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_386\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_387\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_388\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_389\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_390\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_391\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_392\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_396\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_207\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_402\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_397\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_287\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_288\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_289\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_290\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_291\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_292\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_348\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_398\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_399\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_400\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_401\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_635\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_557\", \"type\": {\"type\": \"array\", \"elementType\": \"long\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_382\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_279\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_636\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_207\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_637\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_287\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_288\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_289\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_290\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_291\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_292\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_638\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_565\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_380\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_381\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_382\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_383\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_384\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_385\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_386\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_387\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_388\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_389\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_390\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_391\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_392\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_639\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_301\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_207\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_557\", \"type\": {\"type\": \"array\", \"elementType\": \"long\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_640\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_641\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_642\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_643\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_644\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_645\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_646\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_647\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_648\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_649\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_650\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_651\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_652\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_022\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_653\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_654\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_655\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_656\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_657\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_658\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_659\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_660\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_661\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_662\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_663\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_664\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_665\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_666\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_667\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_668\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_608\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_669\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_608\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_670\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_608\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_671\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_608\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_672\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_673\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_331\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_674\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_675\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_676\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_677\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_678\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_608\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_679\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_608\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_680\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_608\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_681\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_608\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_682\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_608\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_683\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_608\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_684", + "type_text": "struct", + "type_name": "STRUCT", + "position": 25, + "type_json": "{\"name\": \"field_684\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_095\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_279\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_685", + "type_text": "struct", + "type_name": "STRUCT", + "position": 26, + "type_json": "{\"name\": \"field_685\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_686\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_105\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_687\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_688\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_689\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_690\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_691\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_692\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_693\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_694\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_695\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_696\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_697", + "type_text": "string", + "type_name": "STRING", + "position": 27, + "type_json": "{\"name\": \"field_697\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_698", + "type_text": "string", + "type_name": "STRING", + "position": 28, + "type_json": "{\"name\": \"field_698\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_699", + "type_text": "string", + "type_name": "STRING", + "position": 29, + "type_json": "{\"name\": \"field_699\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_700", + "type_text": "boolean", + "type_name": "BOOLEAN", + "position": 30, + "type_json": "{\"name\": \"field_700\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_701", + "type_text": "boolean", + "type_name": "BOOLEAN", + "position": 31, + "type_json": "{\"name\": \"field_701\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_702", + "type_text": "array,field_284:array>>", + "type_name": "ARRAY", + "position": 32, + "type_json": "{\"name\": \"field_702\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_279\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_095\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_280\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_281\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_282\", \"type\": {\"type\": \"array\", \"elementType\": \"long\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_284\", \"type\": {\"type\": \"array\", \"elementType\": \"long\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_703", + "type_text": "bigint", + "type_name": "LONG", + "position": 33, + "type_json": "{\"name\": \"field_703\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_704", + "type_text": "string", + "type_name": "STRING", + "position": 34, + "type_json": "{\"name\": \"field_704\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_705", + "type_text": "string", + "type_name": "STRING", + "position": 35, + "type_json": "{\"name\": \"field_705\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_706", + "type_text": "bigint", + "type_name": "LONG", + "position": 36, + "type_json": "{\"name\": \"field_706\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_707", + "type_text": "bigint", + "type_name": "LONG", + "position": 37, + "type_json": "{\"name\": \"field_707\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_708", + "type_text": "boolean", + "type_name": "BOOLEAN", + "position": 38, + "type_json": "{\"name\": \"field_708\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_709", + "type_text": "boolean", + "type_name": "BOOLEAN", + "position": 39, + "type_json": "{\"name\": \"field_709\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_710", + "type_text": "string", + "type_name": "STRING", + "position": 40, + "type_json": "{\"name\": \"field_710\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_711", + "type_text": "string", + "type_name": "STRING", + "position": 41, + "type_json": "{\"name\": \"field_711\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_712", + "type_text": "boolean", + "type_name": "BOOLEAN", + "position": 42, + "type_json": "{\"name\": \"field_712\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_713", + "type_text": "string", + "type_name": "STRING", + "position": 43, + "type_json": "{\"name\": \"field_713\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_714", + "type_text": "struct>", + "type_name": "STRUCT", + "position": 44, + "type_json": "{\"name\": \"field_714\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_715\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_716\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_717\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_718\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_719\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_720", + "type_text": "boolean", + "type_name": "BOOLEAN", + "position": 45, + "type_json": "{\"name\": \"field_720\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_721", + "type_text": "struct,field_392:string,field_344:struct>,field_726:map>,field_727:array,field_141:struct>>>,field_729:bigint,field_730:string,field_397:struct>>,field_731:struct,field_737:struct,field_738:struct,field_739:bigint>,field_740:string,field_741:array,field_742:array,field_745:struct,field_747:array,field_748:boolean,field_749:array,field_750:array>,field_751:struct,field_752:struct,field_753:array,field_748:boolean,field_749:array,field_754:array>,field_755:string,field_756:string>>,field_757:struct,field_760:array,field_761:array>,field_762:array,field_763:array,field_764:string,field_765:array>>,field_766:boolean,field_402:array,field_767:array>,field_388:string,field_389:string,field_391:string,field_390:string>,field_771:struct,field_778:struct,field_358:bigint>>,field_779:struct,field_358:bigint>>>>,field_766:boolean,field_781:string,field_782:string>,field_783:struct,field_793:struct>,field_381:string,field_794:map,field_728:string,field_795:array>,field_796:struct,field_392:string,field_344:struct>,field_726:map>,field_727:array,field_141:struct>>>,field_729:bigint,field_730:string,field_397:struct>>,field_731:struct,field_737:struct,field_738:struct,field_739:bigint>,field_740:string,field_741:array,field_742:array,field_745:struct,field_747:array,field_748:boolean,field_749:array,field_750:array>,field_751:struct,field_752:struct,field_753:array,field_748:boolean,field_749:array,field_754:array>,field_755:string,field_756:string>>,field_757:struct,field_760:array,field_761:array>,field_762:array,field_763:array,field_764:string,field_765:array>>,field_766:boolean,field_402:array,field_767:array>,field_388:string,field_389:string,field_391:string,field_390:string>>>>,field_800:string,field_801:map,field_782:string,field_802:map,field_803:string,field_804:string,field_805:bigint,field_806:array,field_807:map,field_808:string,field_809:string,field_810:string,field_811:array>,field_814:array>", + "type_name": "STRUCT", + "position": 46, + "type_json": "{\"name\": \"field_721\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_722\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_382\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_637\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_723\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_724\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_629\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_392\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_344\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_348\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_349\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_725\", \"type\": {\"type\": \"map\", \"keyType\": \"string\", \"valueType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_354\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_355\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_356\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_357\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_349\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"valueContainsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_726\", \"type\": {\"type\": \"map\", \"keyType\": \"string\", \"valueType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_354\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_355\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_356\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_357\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_349\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"valueContainsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_727\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_135\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_381\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_728\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_141\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_354\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_355\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_356\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_357\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_349\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_729\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_730\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_397\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_287\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_288\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_289\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_290\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_291\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_292\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_731\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_732\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_733\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_734\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_735\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_736\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_737\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_733\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_734\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_735\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_736\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_738\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_733\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_734\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_735\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_736\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_739\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_740\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_741\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_742\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_743\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_744\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_745\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_746\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_747\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_748\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_749\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_750\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_751\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_746\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_752\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_382\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_383\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_384\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_385\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_386\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_387\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_388\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_389\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_390\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_391\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_753\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_748\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_749\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_754\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_755\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_756\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_757\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_758\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_207\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_383\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_384\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_759\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_386\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_387\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_760\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_761\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_762\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_291\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_758\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_207\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_383\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_384\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_759\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_386\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_387\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_763\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_764\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_765\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_766\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_402\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_767\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_768\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_769\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_770\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_388\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_389\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_391\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_390\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_771\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_637\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_772\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_773\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_740\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_774\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_775\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_776\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_777\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_778\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_382\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_381\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_382\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_383\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_384\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_385\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_386\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_387\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_388\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_389\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_390\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_391\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_358\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_779\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_780\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_381\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_207\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_383\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_384\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_759\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_386\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_387\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_358\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_766\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_781\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_782\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_783\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_784\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_785\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_786\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_787\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_788\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_789\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_790\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_791\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_792\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_793\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_323\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_249\", \"type\": {\"type\": \"map\", \"keyType\": \"string\", \"valueType\": \"string\", \"valueContainsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_381\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_794\", \"type\": {\"type\": \"map\", \"keyType\": \"string\", \"valueType\": \"string\", \"valueContainsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_728\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_795\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_135\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_141\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_796\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_797\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_637\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_798\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_799\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_637\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_723\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_724\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_629\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_392\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_344\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_348\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_349\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_725\", \"type\": {\"type\": \"map\", \"keyType\": \"string\", \"valueType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_354\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_355\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_356\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_357\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_349\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"valueContainsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_726\", \"type\": {\"type\": \"map\", \"keyType\": \"string\", \"valueType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_354\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_355\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_356\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_357\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_349\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"valueContainsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_727\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_135\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_381\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_728\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_141\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_354\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_355\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_356\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_357\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_349\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_729\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_730\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_397\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_287\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_288\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_289\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_290\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_291\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_292\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_731\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_732\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_733\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_734\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_735\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_736\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_737\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_733\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_734\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_735\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_736\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_738\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_733\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_734\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_735\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_736\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_739\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_740\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_741\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_742\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_743\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_744\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_745\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_746\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_747\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_748\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_749\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_750\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_751\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_746\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_752\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_382\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_383\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_384\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_385\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_386\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_387\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_388\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_389\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_390\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_391\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_753\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_748\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_749\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_754\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_755\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_756\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_757\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_758\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_207\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_383\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_384\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_759\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_386\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_387\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_760\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_761\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_762\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_291\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_758\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_207\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_383\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_384\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_759\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_386\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_387\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_763\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_764\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_765\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_766\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_402\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_767\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_768\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_769\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_770\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_388\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_389\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_391\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_390\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_800\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_801\", \"type\": {\"type\": \"map\", \"keyType\": \"string\", \"valueType\": \"string\", \"valueContainsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_782\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_802\", \"type\": {\"type\": \"map\", \"keyType\": \"string\", \"valueType\": \"string\", \"valueContainsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_803\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_804\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_805\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_806\", \"type\": {\"type\": \"array\", \"elementType\": \"long\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_807\", \"type\": {\"type\": \"map\", \"keyType\": \"integer\", \"valueType\": \"string\", \"valueContainsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_808\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_809\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_810\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_811\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_812\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_813\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_814\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_815", + "type_text": "string", + "type_name": "STRING", + "position": 47, + "type_json": "{\"name\": \"field_815\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_816", + "type_text": "struct", + "type_name": "STRUCT", + "position": 48, + "type_json": "{\"name\": \"field_816\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_817\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_818\", \"type\": \"binary\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_819\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_604\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_820\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_821\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_822\", \"type\": \"binary\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_823", + "type_text": "boolean", + "type_name": "BOOLEAN", + "position": 49, + "type_json": "{\"name\": \"field_823\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_824", + "type_text": "boolean", + "type_name": "BOOLEAN", + "position": 50, + "type_json": "{\"name\": \"field_824\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_825", + "type_text": "struct", + "type_name": "STRUCT", + "position": 51, + "type_json": "{\"name\": \"field_825\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_104\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_826", + "type_text": "array", + "type_name": "ARRAY", + "position": 52, + "type_json": "{\"name\": \"field_826\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_827", + "type_text": "array,field_835:struct>,field_840:array>>>,field_843:int,field_844:boolean>,field_845:string,field_846:string,field_847:array>,field_850:int>>", + "type_name": "ARRAY", + "position": 53, + "type_json": "{\"name\": \"field_827\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_827\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_817\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_828\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_829\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_830\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_831\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_012\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_832\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_042\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_092\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_038\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_833\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_834\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_010\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_043\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_835\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_032\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_033\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_013\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_836\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_837\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_838\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_839\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_840\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_841\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_842\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_843\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_844\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_845\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_846\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_847\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_841\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_843\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_848\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_849\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_850\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_851", + "type_text": "struct,field_853:string,field_854:bigint,field_855:bigint,field_856:array>,field_859:string,field_860:string>", + "type_name": "STRUCT", + "position": 54, + "type_json": "{\"name\": \"field_851\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_852\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_284\", \"type\": {\"type\": \"array\", \"elementType\": \"long\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_853\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_854\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_855\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_856\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_857\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_464\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_465\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_858\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_859\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_860\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_861", + "type_text": "struct", + "type_name": "STRUCT", + "position": 55, + "type_json": "{\"name\": \"field_861\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_817\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_818\", \"type\": \"binary\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_819\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_604\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_820\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_821\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_822\", \"type\": \"binary\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_862", + "type_text": "string", + "type_name": "STRING", + "position": 56, + "type_json": "{\"name\": \"field_862\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_863", + "type_text": "string", + "type_name": "STRING", + "position": 57, + "type_json": "{\"name\": \"field_863\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_864", + "type_text": "boolean", + "type_name": "BOOLEAN", + "position": 58, + "type_json": "{\"name\": \"field_864\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_865", + "type_text": "int", + "type_name": "INT", + "position": 59, + "type_json": "{\"name\": \"field_865\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_866", + "type_text": "boolean", + "type_name": "BOOLEAN", + "position": 60, + "type_json": "{\"name\": \"field_866\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_867", + "type_text": "string", + "type_name": "STRING", + "position": 61, + "type_json": "{\"name\": \"field_867\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_035", + "type_text": "string", + "type_name": "STRING", + "position": 62, + "type_json": "{\"name\": \"field_035\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_868", + "type_text": "string", + "type_name": "STRING", + "position": 63, + "type_json": "{\"name\": \"field_868\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_869", + "type_text": "string", + "type_name": "STRING", + "position": 64, + "type_json": "{\"name\": \"field_869\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_870", + "type_text": "string", + "type_name": "STRING", + "position": 65, + "type_json": "{\"name\": \"field_870\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_871", + "type_text": "string", + "type_name": "STRING", + "position": 66, + "type_json": "{\"name\": \"field_871\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_872", + "type_text": "string", + "type_name": "STRING", + "position": 67, + "type_json": "{\"name\": \"field_872\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_873", + "type_text": "string", + "type_name": "STRING", + "position": 68, + "type_json": "{\"name\": \"field_873\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_036", + "type_text": "string", + "type_name": "STRING", + "position": 69, + "type_json": "{\"name\": \"field_036\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_874", + "type_text": "string", + "type_name": "STRING", + "position": 70, + "type_json": "{\"name\": \"field_874\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_875", + "type_text": "boolean", + "type_name": "BOOLEAN", + "position": 71, + "type_json": "{\"name\": \"field_875\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_876", + "type_text": "string", + "type_name": "STRING", + "position": 72, + "type_json": "{\"name\": \"field_876\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_877", + "type_text": "string", + "type_name": "STRING", + "position": 73, + "type_json": "{\"name\": \"field_877\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_878", + "type_text": "boolean", + "type_name": "BOOLEAN", + "position": 74, + "type_json": "{\"name\": \"field_878\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_879", + "type_text": "string", + "type_name": "STRING", + "position": 75, + "type_json": "{\"name\": \"field_879\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_880", + "type_text": "string", + "type_name": "STRING", + "position": 76, + "type_json": "{\"name\": \"field_880\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_881", + "type_text": "boolean", + "type_name": "BOOLEAN", + "position": 77, + "type_json": "{\"name\": \"field_881\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_882", + "type_text": "struct", + "type_name": "STRUCT", + "position": 78, + "type_json": "{\"name\": \"field_882\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_883\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_884\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_885", + "type_text": "array,field_347:struct>>,field_305:boolean,field_302:bigint,field_306:array>,field_353:struct>>,field_359:string,field_360:string,field_361:bigint,field_362:boolean>,field_857:string>>", + "type_name": "ARRAY", + "position": 79, + "type_json": "{\"name\": \"field_885\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_369\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_370\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_371\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_372\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_373\", \"type\": \"double\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_374\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_347\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_348\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_349\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_350\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_351\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_352\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_207\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_286\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_287\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_288\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_289\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_290\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_291\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_292\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_305\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_302\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_306\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_353\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_354\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_355\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_356\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_357\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_358\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_359\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_360\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_361\", \"type\": \"long\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_362\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_857\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_886", + "type_text": "string", + "type_name": "STRING", + "position": 80, + "type_json": "{\"name\": \"field_886\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_887", + "type_text": "string", + "type_name": "STRING", + "position": 81, + "type_json": "{\"name\": \"field_887\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_888", + "type_text": "string", + "type_name": "STRING", + "position": 82, + "type_json": "{\"name\": \"field_888\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_889", + "type_text": "struct>>", + "type_name": "STRUCT", + "position": 83, + "type_json": "{\"name\": \"field_889\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_105\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_106\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_107\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_108\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_109\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_110\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_105\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_106\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_890", + "type_text": "struct>,field_901:boolean>", + "type_name": "STRUCT", + "position": 84, + "type_json": "{\"name\": \"field_890\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_891\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_815\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_892\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_893\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_803\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_894\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_895\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_896\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_897\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_898\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_899\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_900\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_901\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_902", + "type_text": "struct,field_912:string>>", + "type_name": "STRUCT", + "position": 85, + "type_json": "{\"name\": \"field_902\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_903\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_904\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_905\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_906\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_907\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_908\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_909\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_910\", \"type\": \"boolean\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_911\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_912\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_251", + "type_text": "string", + "type_name": "STRING", + "position": 86, + "type_json": "{\"name\": \"field_251\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_913", + "type_text": "array", + "type_name": "ARRAY", + "position": 87, + "type_json": "{\"name\": \"field_913\", \"type\": {\"type\": \"array\", \"elementType\": \"string\", \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_914", + "type_text": "string", + "type_name": "STRING", + "position": 88, + "type_json": "{\"name\": \"field_914\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_915", + "type_text": "array>", + "type_name": "ARRAY", + "position": 89, + "type_json": "{\"name\": \"field_915\", \"type\": {\"type\": \"array\", \"elementType\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_916\", \"type\": \"integer\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_917\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_918\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_919\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_920\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"containsNull\": true}, \"nullable\": true, \"metadata\": {}}", + "nullable": true + }, + { + "name": "field_921", + "type_text": "struct", + "type_name": "STRUCT", + "position": 90, + "type_json": "{\"name\": \"field_921\", \"type\": {\"type\": \"struct\", \"fields\": [{\"name\": \"field_922\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_923\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_924\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}, {\"name\": \"field_925\", \"type\": \"string\", \"nullable\": true, \"metadata\": {}}]}, \"nullable\": true, \"metadata\": {}}", + "nullable": true + } + ] +} \ No newline at end of file diff --git a/src/sinks/databricks_zerobus/tests/fixtures/mixed_types_schema.json b/src/sinks/databricks_zerobus/tests/fixtures/mixed_types_schema.json new file mode 100644 index 0000000000000..0966ab4fe9048 --- /dev/null +++ b/src/sinks/databricks_zerobus/tests/fixtures/mixed_types_schema.json @@ -0,0 +1,47 @@ +{ + "name": "mixed_types_table", + "catalog_name": "test_catalog", + "schema_name": "test_schema", + "columns": [ + { + "name": "field_001", + "type_text": "timestamp", + "type_name": "TIMESTAMP", + "position": 0, + "nullable": true, + "type_json": "{\"name\":\"field_001\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}}" + }, + { + "name": "field_002", + "type_text": "bigint", + "type_name": "LONG", + "position": 12, + "nullable": true, + "type_json": "{\"name\":\"field_002\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}" + }, + { + "name": "field_003", + "type_text": "struct>", + "type_name": "STRUCT", + "position": 13, + "nullable": true, + "type_json": "{\"name\":\"field_003\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"field_004\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"field_005\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"field_006\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}" + }, + { + "name": "field_007", + "type_text": "array", + "type_name": "ARRAY", + "position": 16, + "nullable": true, + "type_json": "{\"name\":\"field_007\",\"type\":{\"type\":\"array\",\"elementType\":\"long\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}}" + }, + { + "name": "field_008", + "type_text": "map", + "type_name": "MAP", + "position": 20, + "nullable": true, + "type_json": "{\"name\":\"field_008\",\"type\":{\"type\":\"map\",\"keyType\":\"string\",\"valueType\":\"string\",\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{}}" + } + ] +} diff --git a/src/sinks/databricks_zerobus/tests/fixtures/nested_struct_schema.json b/src/sinks/databricks_zerobus/tests/fixtures/nested_struct_schema.json new file mode 100644 index 0000000000000..69d674fc5e6e4 --- /dev/null +++ b/src/sinks/databricks_zerobus/tests/fixtures/nested_struct_schema.json @@ -0,0 +1,19 @@ +{ + "name": "nested_struct_table", + "catalog_name": "test_catalog", + "schema_name": "test_schema", + "table_type": "MANAGED", + "data_source_format": "DELTA", + "columns": [ + { + "name": "user_info", + "type_text": "struct>", + "type_name": "STRUCT", + "position": 1, + "type_precision": 0, + "type_scale": 0, + "nullable": true, + "type_json": "{\"name\":\"user_info\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"address\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"street\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"city\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"zipcode\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}" + } + ] +} diff --git a/src/sinks/databricks_zerobus/tests/fixtures/nested_structs_complete_schema.json b/src/sinks/databricks_zerobus/tests/fixtures/nested_structs_complete_schema.json new file mode 100644 index 0000000000000..f30f749d13230 --- /dev/null +++ b/src/sinks/databricks_zerobus/tests/fixtures/nested_structs_complete_schema.json @@ -0,0 +1,127 @@ +{ + "name": "nested_structs_table", + "catalog_name": "test_catalog", + "schema_name": "test_schema", + "columns": [ + { + "name": "field_001", + "type_text": "bigint", + "type_name": "LONG", + "position": 0, + "nullable": true, + "type_json": "{\"name\":\"field_001\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}" + }, + { + "name": "field_002", + "type_text": "string", + "type_name": "STRING", + "position": 1, + "nullable": true, + "type_json": "{\"name\":\"field_002\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}" + }, + { + "name": "field_003", + "type_text": "string", + "type_name": "STRING", + "position": 2, + "nullable": true, + "type_json": "{\"name\":\"field_003\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}" + }, + { + "name": "field_004", + "type_text": "string", + "type_name": "STRING", + "position": 3, + "nullable": true, + "type_json": "{\"name\":\"field_004\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}" + }, + { + "name": "field_005", + "type_text": "string", + "type_name": "STRING", + "position": 4, + "nullable": true, + "type_json": "{\"name\":\"field_005\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}" + }, + { + "name": "field_006", + "type_text": "bigint", + "type_name": "LONG", + "position": 5, + "nullable": true, + "type_json": "{\"name\":\"field_006\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}" + }, + { + "name": "field_007", + "type_text": "bigint", + "type_name": "LONG", + "position": 6, + "nullable": true, + "type_json": "{\"name\":\"field_007\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}" + }, + { + "name": "field_008", + "type_text": "struct,field_012:struct,field_014:struct,field_016:struct>", + "type_name": "STRUCT", + "position": 7, + "nullable": true, + "type_json": "{\"name\":\"field_008\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"field_009\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"field_010\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"field_011\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"field_012\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"field_013\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"field_014\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"field_015\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"field_016\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"field_017\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}" + }, + { + "name": "field_018", + "type_text": "struct", + "type_name": "STRUCT", + "position": 8, + "nullable": true, + "type_json": "{\"name\":\"field_018\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"field_019\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"field_020\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}" + }, + { + "name": "field_021", + "type_text": "struct", + "type_name": "STRUCT", + "position": 9, + "nullable": true, + "type_json": "{\"name\":\"field_021\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"field_022\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"field_023\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"field_020\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"field_024\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"field_025\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"field_026\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}" + }, + { + "name": "field_027", + "type_text": "array", + "type_name": "ARRAY", + "position": 10, + "nullable": true, + "type_json": "{\"name\":\"field_027\",\"type\":{\"type\":\"array\",\"elementType\":\"long\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}}" + }, + { + "name": "field_028", + "type_text": "boolean", + "type_name": "BOOLEAN", + "position": 11, + "nullable": true, + "type_json": "{\"name\":\"field_028\",\"type\":\"boolean\",\"nullable\":true,\"metadata\":{}}" + }, + { + "name": "field_029", + "type_text": "string", + "type_name": "STRING", + "position": 12, + "nullable": true, + "type_json": "{\"name\":\"field_029\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}" + }, + { + "name": "field_030", + "type_text": "string", + "type_name": "STRING", + "position": 13, + "nullable": true, + "type_json": "{\"name\":\"field_030\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}" + }, + { + "name": "field_031", + "type_text": "boolean", + "type_name": "BOOLEAN", + "position": 14, + "nullable": true, + "type_json": "{\"name\":\"field_031\",\"type\":\"boolean\",\"nullable\":true,\"metadata\":{}}" + } + ] +} diff --git a/src/sinks/databricks_zerobus/tests/fixtures/simple_table_schema.json b/src/sinks/databricks_zerobus/tests/fixtures/simple_table_schema.json new file mode 100644 index 0000000000000..6ccef8bb97e43 --- /dev/null +++ b/src/sinks/databricks_zerobus/tests/fixtures/simple_table_schema.json @@ -0,0 +1,49 @@ +{ + "name": "simple_table", + "catalog_name": "test_catalog", + "schema_name": "test_schema", + "table_type": "MANAGED", + "data_source_format": "DELTA", + "columns": [ + { + "name": "id", + "type_text": "bigint", + "type_name": "BIGINT", + "position": 1, + "type_precision": 0, + "type_scale": 0, + "nullable": false, + "type_json": "{\"name\":\"id\",\"type\":\"long\",\"nullable\":false,\"metadata\":{}}" + }, + { + "name": "name", + "type_text": "string", + "type_name": "STRING", + "position": 2, + "type_precision": 0, + "type_scale": 0, + "nullable": true, + "type_json": "{\"name\":\"name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}" + }, + { + "name": "created_at", + "type_text": "timestamp", + "type_name": "TIMESTAMP", + "position": 3, + "type_precision": 0, + "type_scale": 0, + "nullable": true, + "type_json": "{\"name\":\"created_at\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}}" + }, + { + "name": "is_active", + "type_text": "boolean", + "type_name": "BOOLEAN", + "position": 4, + "type_precision": 0, + "type_scale": 0, + "nullable": true, + "type_json": "{\"name\":\"is_active\",\"type\":\"boolean\",\"nullable\":true,\"metadata\":{}}" + } + ] +} diff --git a/src/sinks/databricks_zerobus/unity_catalog_schema.rs b/src/sinks/databricks_zerobus/unity_catalog_schema.rs new file mode 100644 index 0000000000000..1051e19b40ad5 --- /dev/null +++ b/src/sinks/databricks_zerobus/unity_catalog_schema.rs @@ -0,0 +1,1812 @@ +//! Unity Catalog schema fetching and protobuf descriptor generation. + +use bytes::Buf; +use http::{Request, Uri}; +use http_body::Body as HttpBody; +use hyper::Body; +use percent_encoding::{NON_ALPHANUMERIC, percent_encode}; +use serde::Deserialize; +use serde_json::Value as JsonValue; +use prost_reflect::prost_types; + +use super::error::ZerobusSinkError; +use crate::config::ProxyConfig; +use crate::http::HttpClient; +use crate::tls::TlsSettings; + +/// Unity Catalog table column information +#[derive(Debug, Deserialize, Clone)] +pub struct UnityCatalogColumn { + pub name: String, + #[allow(dead_code)] // Will be used for complex type parsing + pub type_text: String, + pub type_name: String, + #[serde(default)] + pub position: i32, + pub nullable: bool, + #[allow(dead_code)] // Will be used for complex type parsing + #[serde(default)] + pub type_json: String, +} + +/// Unity Catalog table schema response +#[derive(Debug, Deserialize)] +pub struct UnityCatalogTableSchema { + pub name: String, + pub catalog_name: String, + pub schema_name: String, + pub columns: Vec, +} + +/// OAuth token response from Databricks +#[derive(Debug, Deserialize)] +struct OAuthTokenResponse { + access_token: String, +} + +/// Represents a parsed complex type from type_json +#[derive(Debug, Clone)] +enum ComplexType { + Primitive(PrimitiveType), + Struct(StructType), + Array(Box), + Map { + key_type: Box, + #[allow(dead_code)] // Will be used for full MAP support + value_type: Box, + }, +} + +/// Primitive types +#[derive(Debug, Clone)] +enum PrimitiveType { + String, + Long, + Integer, + Short, + Byte, + Double, + Float, + Boolean, + Binary, + Timestamp, + Date, + Decimal { _precision: i32, _scale: i32 }, +} + +/// Struct field definition from type_json +#[derive(Debug, Clone)] +struct StructField { + name: String, + field_type: ComplexType, + nullable: bool, +} + +/// Struct type definition +#[derive(Debug, Clone)] +struct StructType { + fields: Vec, +} + +/// Fetch table schema from Unity Catalog API +pub async fn fetch_table_schema( + unity_catalog_endpoint: &str, + table_name: &str, + client_id: &str, + client_secret: &str, +) -> Result { + // First, get OAuth token + let token = get_oauth_token(unity_catalog_endpoint, client_id, client_secret).await?; + + // Fetch table schema + let url = format!( + "{}/api/2.0/unity-catalog/tables/{}", + unity_catalog_endpoint.trim_end_matches('/'), + table_name + ); + + let uri: Uri = url.parse().map_err(|e| ZerobusSinkError::ConfigError { + message: format!("Invalid Unity Catalog endpoint URL: {}", e), + })?; + + let http_client = + HttpClient::new(TlsSettings::default(), &ProxyConfig::default()).map_err(|e| { + ZerobusSinkError::ConfigError { + message: format!("Failed to create HTTP client: {}", e), + } + })?; + + let request = Request::get(uri) + .header("Authorization", format!("Bearer {}", token)) + .header("Content-Type", "application/json") + .body(Body::empty()) + .map_err(|e| ZerobusSinkError::ConfigError { + message: format!("Failed to build request: {}", e), + })?; + + let response = http_client + .send(request) + .await + .map_err(|e| ZerobusSinkError::ConfigError { + message: format!("Failed to fetch table schema: {}", e), + })?; + + let status = response.status(); + if !status.is_success() { + let body_bytes = response + .into_body() + .collect() + .await + .map(|c| c.to_bytes()) + .unwrap_or_default(); + let error_text = String::from_utf8_lossy(&body_bytes); + return Err(ZerobusSinkError::ConfigError { + message: format!( + "Unity Catalog API returned error {}: {}", + status, error_text + ), + }); + } + + let body_bytes = response + .into_body() + .collect() + .await + .map(|c| c.to_bytes()) + .map_err(|e| ZerobusSinkError::ConfigError { + message: format!("Failed to read response body: {}", e), + })?; + + let schema: UnityCatalogTableSchema = + serde_json::from_reader(body_bytes.reader()).map_err(|e| { + ZerobusSinkError::ConfigError { + message: format!("Failed to parse table schema response: {}", e), + } + })?; + + Ok(schema) +} + +/// Get OAuth token from Databricks +async fn get_oauth_token( + unity_catalog_endpoint: &str, + client_id: &str, + client_secret: &str, +) -> Result { + let token_url = format!( + "{}/oidc/v1/token", + unity_catalog_endpoint.trim_end_matches('/') + ); + + let uri: Uri = token_url + .parse() + .map_err(|e| ZerobusSinkError::ConfigError { + message: format!("Invalid token endpoint URL: {}", e), + })?; + + // Build form-encoded body + let form_body = format!( + "grant_type=client_credentials&client_id={}&client_secret={}&scope=all-apis", + percent_encode(client_id.as_bytes(), NON_ALPHANUMERIC), + percent_encode(client_secret.as_bytes(), NON_ALPHANUMERIC) + ); + + let http_client = + HttpClient::new(TlsSettings::default(), &ProxyConfig::default()).map_err(|e| { + ZerobusSinkError::ConfigError { + message: format!("Failed to create HTTP client: {}", e), + } + })?; + + let request = Request::post(uri) + .header("Content-Type", "application/x-www-form-urlencoded") + .body(Body::from(form_body)) + .map_err(|e| ZerobusSinkError::ConfigError { + message: format!("Failed to build OAuth request: {}", e), + })?; + + let response = http_client + .send(request) + .await + .map_err(|e| ZerobusSinkError::ConfigError { + message: format!("Failed to get OAuth token: {}", e), + })?; + + let status = response.status(); + if !status.is_success() { + let body_bytes = response + .into_body() + .collect() + .await + .map(|c| c.to_bytes()) + .unwrap_or_default(); + let error_text = String::from_utf8_lossy(&body_bytes); + return Err(ZerobusSinkError::ConfigError { + message: format!("OAuth token request failed {}: {}", status, error_text), + }); + } + + let body_bytes = response + .into_body() + .collect() + .await + .map(|c| c.to_bytes()) + .map_err(|e| ZerobusSinkError::ConfigError { + message: format!("Failed to read OAuth response body: {}", e), + })?; + + let token_response: OAuthTokenResponse = + serde_json::from_reader(body_bytes.reader()).map_err(|e| { + ZerobusSinkError::ConfigError { + message: format!("Failed to parse OAuth token response: {}", e), + } + })?; + + Ok(token_response.access_token) +} + +/// Parse type_json string into ComplexType +fn parse_type_json(type_json: &str) -> Result { + if type_json.is_empty() || type_json == "{}" { + return Err(ZerobusSinkError::ConfigError { + message: "Empty type_json".to_string(), + }); + } + + let json: JsonValue = + serde_json::from_str(type_json).map_err(|e| ZerobusSinkError::ConfigError { + message: format!("Failed to parse type_json: {}", e), + })?; + + // Unity Catalog wraps types in {"name": "field_name", "type": {...}} + // Check if this is the wrapped format by looking for both "name" and "type" fields + let type_json = if let Some(obj) = json.as_object() { + if obj.contains_key("name") && obj.contains_key("type") { + // This is Unity Catalog wrapped format - extract the inner "type" + obj.get("type").unwrap() + } else { + // This is a direct type definition + &json + } + } else { + &json + }; + + parse_complex_type(type_json) +} + +/// Recursively parse a complex type from JSON +fn parse_complex_type(json: &JsonValue) -> Result { + // Handle simple string types (for nested fields) + if let Some(type_str) = json.as_str() { + return parse_primitive_type(type_str); + } + + // Handle type object + let type_obj = json + .as_object() + .ok_or_else(|| ZerobusSinkError::ConfigError { + message: format!("Expected type object, got: {:?}", json), + })?; + + // Get the "type" field + let type_field = type_obj + .get("type") + .and_then(|v| v.as_str()) + .ok_or_else(|| ZerobusSinkError::ConfigError { + message: format!("Missing 'type' field in type_json: {:?}", type_obj), + })?; + + match type_field { + "struct" => parse_struct_type(type_obj), + "array" => parse_array_type(type_obj), + "map" => parse_map_type(type_obj), + primitive => parse_primitive_type(primitive), + } +} + +/// Parse primitive type string +fn parse_primitive_type(type_str: &str) -> Result { + let primitive = match type_str { + "string" => PrimitiveType::String, + "long" => PrimitiveType::Long, + "integer" => PrimitiveType::Integer, + "short" => PrimitiveType::Short, + "byte" => PrimitiveType::Byte, + "double" => PrimitiveType::Double, + "float" => PrimitiveType::Float, + "boolean" => PrimitiveType::Boolean, + "binary" => PrimitiveType::Binary, + "timestamp" => PrimitiveType::Timestamp, + "date" => PrimitiveType::Date, + other if other.starts_with("decimal") => { + // Parse decimal(precision, scale) + PrimitiveType::Decimal { + _precision: 38, + _scale: 10, + } // Default values + } + unknown => { + return Err(ZerobusSinkError::ConfigError { + message: format!("Unknown primitive type: {}", unknown), + }); + } + }; + Ok(ComplexType::Primitive(primitive)) +} + +/// Parse STRUCT type +fn parse_struct_type( + type_obj: &serde_json::Map, +) -> Result { + let fields_json = type_obj + .get("fields") + .and_then(|v| v.as_array()) + .ok_or_else(|| ZerobusSinkError::ConfigError { + message: "STRUCT type missing 'fields' array".to_string(), + })?; + + let mut fields = Vec::new(); + for field_json in fields_json { + let field_obj = field_json + .as_object() + .ok_or_else(|| ZerobusSinkError::ConfigError { + message: format!("Expected field object, got: {:?}", field_json), + })?; + + let name = field_obj + .get("name") + .and_then(|v| v.as_str()) + .ok_or_else(|| ZerobusSinkError::ConfigError { + message: "Field missing 'name'".to_string(), + })? + .to_string(); + + let nullable = field_obj + .get("nullable") + .and_then(|v| v.as_bool()) + .unwrap_or(true); + + // Parse the field type (can be nested) + let field_type_json = + field_obj + .get("type") + .ok_or_else(|| ZerobusSinkError::ConfigError { + message: format!("Field '{}' missing 'type'", name), + })?; + + let field_type = parse_complex_type(field_type_json)?; + + fields.push(StructField { + name, + field_type, + nullable, + }); + } + + Ok(ComplexType::Struct(StructType { fields })) +} + +/// Parse ARRAY type +fn parse_array_type( + type_obj: &serde_json::Map, +) -> Result { + let element_type_json = + type_obj + .get("elementType") + .ok_or_else(|| ZerobusSinkError::ConfigError { + message: "ARRAY type missing 'elementType'".to_string(), + })?; + + let element_type = parse_complex_type(element_type_json)?; + Ok(ComplexType::Array(Box::new(element_type))) +} + +/// Parse MAP type +fn parse_map_type( + type_obj: &serde_json::Map, +) -> Result { + let key_type_json = type_obj + .get("keyType") + .ok_or_else(|| ZerobusSinkError::ConfigError { + message: "MAP type missing 'keyType'".to_string(), + })?; + + let value_type_json = + type_obj + .get("valueType") + .ok_or_else(|| ZerobusSinkError::ConfigError { + message: "MAP type missing 'valueType'".to_string(), + })?; + + let key_type = parse_complex_type(key_type_json)?; + let value_type = parse_complex_type(value_type_json)?; + + Ok(ComplexType::Map { + key_type: Box::new(key_type), + value_type: Box::new(value_type), + }) +} + +/// Helper structure to collect nested message types during generation +struct MessageCollector { + /// All nested message definitions + nested_messages: Vec, +} + +impl MessageCollector { + const fn new() -> Self { + Self { + nested_messages: Vec::new(), + } + } + + /// Add a nested message definition + fn add_message(&mut self, message: prost_types::DescriptorProto) { + self.nested_messages.push(message); + } +} + +/// Format a protobuf MessageDescriptor as a .proto file string for logging +fn format_descriptor_as_proto(descriptor: &prost_reflect::MessageDescriptor) -> String { + let mut output = String::new(); + format_message_as_proto(descriptor, &mut output, 0); + output +} + +/// Recursively format a message and its nested types +fn format_message_as_proto( + descriptor: &prost_reflect::MessageDescriptor, + output: &mut String, + indent_level: usize, +) { + let indent = " ".repeat(indent_level); + + // Write message header + output.push_str(&format!("{}message {} {{\n", indent, descriptor.name())); + + // Write fields + for field in descriptor.fields() { + let field_indent = " ".repeat(indent_level + 1); + let field_type = format_field_type(&field); + let field_number = field.number(); + output.push_str(&format!( + "{}{}{} = {};\n", + field_indent, + field_type, + field.name(), + field_number + )); + } + + output.push_str(&format!("{}}}\n", indent)); + + // Write nested message types + for nested in descriptor.child_messages() { + output.push('\n'); + format_message_as_proto(&nested, output, indent_level); + } +} + +/// Format a field's type declaration +fn format_field_type(field: &prost_reflect::FieldDescriptor) -> String { + use prost_reflect::Kind; + + if field.is_map() { + // Map fields: map field_name + if let Kind::Message(map_entry) = field.kind() { + let key_field = map_entry.fields().find(|f| f.name() == "key").unwrap(); + let value_field = map_entry.fields().find(|f| f.name() == "value").unwrap(); + let key_type = format_scalar_type(&key_field); + let value_type = format_scalar_type(&value_field); + return format!("map<{}, {}> ", key_type, value_type); + } + } + + let base_type = match field.kind() { + Kind::Message(msg) => msg.name().to_string(), + kind => format_kind_type(&kind), + }; + + if field.is_list() { + format!("repeated {} ", base_type) + } else { + format!("{} ", base_type) + } +} + +/// Format a scalar field type (for map keys/values) +fn format_scalar_type(field: &prost_reflect::FieldDescriptor) -> String { + match field.kind() { + prost_reflect::Kind::Message(msg) => msg.name().to_string(), + kind => format_kind_type(&kind), + } +} + +/// Map Kind enum to proto type string +fn format_kind_type(kind: &prost_reflect::Kind) -> String { + use prost_reflect::Kind; + match kind { + Kind::Double => "double".to_string(), + Kind::Float => "float".to_string(), + Kind::Int32 => "int32".to_string(), + Kind::Int64 => "int64".to_string(), + Kind::Uint32 => "uint32".to_string(), + Kind::Uint64 => "uint64".to_string(), + Kind::Sint32 => "sint32".to_string(), + Kind::Sint64 => "sint64".to_string(), + Kind::Fixed32 => "fixed32".to_string(), + Kind::Fixed64 => "fixed64".to_string(), + Kind::Sfixed32 => "sfixed32".to_string(), + Kind::Sfixed64 => "sfixed64".to_string(), + Kind::Bool => "bool".to_string(), + Kind::String => "string".to_string(), + Kind::Bytes => "bytes".to_string(), + Kind::Message(msg) => msg.name().to_string(), + Kind::Enum(e) => e.name().to_string(), + } +} + +/// Generate protobuf descriptor from Unity Catalog table schema +pub fn generate_descriptor_from_schema( + schema: &UnityCatalogTableSchema, +) -> Result { + let mut proto_fields = Vec::new(); + let mut collector = MessageCollector::new(); + + // Sort columns by position to maintain stable field numbers + let mut columns = schema.columns.clone(); + columns.sort_by_key(|c| c.position); + + for column in columns { + // Skip columns with invalid positions (position should be >= 0) + if column.position < 0 { + continue; + } + + // Try to parse complex types from type_json + let (field_type, type_name, is_repeated) = if column.type_name == "STRUCT" + || column.type_name == "ARRAY" + || column.type_name == "MAP" + { + // Parse type_json for complex types - STRICT MODE: fail on parse errors + let complex_type = + parse_type_json(&column.type_json).map_err(|e| ZerobusSinkError::ConfigError { + message: format!( + "Failed to parse complex type for column '{}': {}. \ + Vector requires all types to be supported. \ + Options: 1) Update Vector to latest version, \ + 2) Use explicit .proto schema file", + column.name, e + ), + })?; + + let is_repeated = matches!( + complex_type, + ComplexType::Array(_) | ComplexType::Map { .. } + ); + let path_prefix = column.name.clone(); + let (field_type, type_name) = + map_complex_type_to_protobuf(&complex_type, &path_prefix, &mut collector)?; + (field_type, type_name, is_repeated) + } else { + // Simple types + let field_type = map_simple_databricks_type(&column.type_name)?; + (field_type, None, false) + }; + + // Determine label based on type + let label = if is_repeated { + // ARRAYs and MAPs are represented as repeated fields + prost_types::field_descriptor_proto::Label::Repeated as i32 + } else if column.nullable { + prost_types::field_descriptor_proto::Label::Optional as i32 + } else { + prost_types::field_descriptor_proto::Label::Required as i32 + }; + + proto_fields.push(prost_types::FieldDescriptorProto { + name: Some(column.name.clone()), + number: Some(column.position + 1), // Protobuf field numbers start at 1, not 0 + label: Some(label), + r#type: Some(field_type as i32), + type_name, + extendee: None, + default_value: None, + oneof_index: None, + json_name: Some(column.name.clone()), + options: None, + proto3_optional: Some(column.nullable && !is_repeated), + }); + } + + // Create the message descriptor + let message_name = format!("{}_{}", schema.schema_name, schema.name); + let message_proto = prost_types::DescriptorProto { + name: Some(message_name.clone()), + field: proto_fields, + extension: vec![], + nested_type: collector.nested_messages, + enum_type: vec![], + extension_range: vec![], + oneof_decl: vec![], + options: None, + reserved_range: vec![], + reserved_name: vec![], + }; + + let file_proto = prost_types::FileDescriptorProto { + name: Some(format!("{}.proto", message_name)), + package: Some(schema.catalog_name.clone()), + message_type: vec![message_proto.clone()], + ..Default::default() + }; + + let file_descriptor_set = prost_types::FileDescriptorSet { + file: vec![file_proto], + }; + + // Build a FileDescriptor + let pool = prost_reflect::DescriptorPool::from_file_descriptor_set(file_descriptor_set) + .map_err(|e| ZerobusSinkError::ConfigError { + message: format!("Failed to build descriptor pool: {}", e), + })?; + + let full_message_name = format!("{}.{}", schema.catalog_name, message_name); + let message_descriptor = pool + .get_message_by_name(&full_message_name) + .ok_or_else(|| ZerobusSinkError::ConfigError { + message: format!("Failed to get message descriptor for {}", full_message_name), + })?; + + // Log the inferred protobuf schema + let proto_schema = format_descriptor_as_proto(&message_descriptor); + info!( + "Inferred protobuf schema from Unity Catalog table {}.{}.{}:\n{}", + schema.catalog_name, schema.schema_name, schema.name, proto_schema + ); + + Ok(message_descriptor) +} + +/// Map simple Databricks type name to protobuf type +fn map_simple_databricks_type( + type_name: &str, +) -> Result { + match type_name { + "STRING" => Ok(prost_types::field_descriptor_proto::Type::String), + "INT" => Ok(prost_types::field_descriptor_proto::Type::Int32), + "LONG" | "BIGINT" => Ok(prost_types::field_descriptor_proto::Type::Int64), + "BOOLEAN" | "BOOL" => Ok(prost_types::field_descriptor_proto::Type::Bool), + "DOUBLE" => Ok(prost_types::field_descriptor_proto::Type::Double), + "FLOAT" => Ok(prost_types::field_descriptor_proto::Type::Float), + "TIMESTAMP" => Ok(prost_types::field_descriptor_proto::Type::Int64), // Unix timestamp in microseconds + "DATE" => Ok(prost_types::field_descriptor_proto::Type::String), + "BINARY" => Ok(prost_types::field_descriptor_proto::Type::Bytes), + "DECIMAL" => Ok(prost_types::field_descriptor_proto::Type::String), + + unknown => Err(ZerobusSinkError::ConfigError { + message: format!("Unsupported Databricks type: {}", unknown), + }), + } +} + +/// Map primitive type to protobuf type +const fn map_primitive_to_protobuf( + primitive: &PrimitiveType, +) -> prost_types::field_descriptor_proto::Type { + match primitive { + PrimitiveType::String => prost_types::field_descriptor_proto::Type::String, + PrimitiveType::Long => prost_types::field_descriptor_proto::Type::Int64, + PrimitiveType::Integer => prost_types::field_descriptor_proto::Type::Int32, + PrimitiveType::Short => prost_types::field_descriptor_proto::Type::Int32, + PrimitiveType::Byte => prost_types::field_descriptor_proto::Type::Int32, + PrimitiveType::Double => prost_types::field_descriptor_proto::Type::Double, + PrimitiveType::Float => prost_types::field_descriptor_proto::Type::Float, + PrimitiveType::Boolean => prost_types::field_descriptor_proto::Type::Bool, + PrimitiveType::Binary => prost_types::field_descriptor_proto::Type::Bytes, + PrimitiveType::Timestamp => prost_types::field_descriptor_proto::Type::String, + PrimitiveType::Date => prost_types::field_descriptor_proto::Type::String, + PrimitiveType::Decimal { .. } => prost_types::field_descriptor_proto::Type::String, + } +} + +/// Map complex type to protobuf, generating nested messages as needed +/// Returns (field_type, optional_type_name) +fn map_complex_type_to_protobuf( + complex_type: &ComplexType, + path_prefix: &str, + collector: &mut MessageCollector, +) -> Result<(prost_types::field_descriptor_proto::Type, Option), ZerobusSinkError> { + match complex_type { + ComplexType::Primitive(primitive) => { + let proto_type = map_primitive_to_protobuf(primitive); + Ok((proto_type, None)) + } + + ComplexType::Struct(struct_type) => { + // Generate a nested message for this struct + let message_name = sanitize_message_name(path_prefix); + let message_proto = generate_struct_message(&message_name, struct_type, collector)?; + collector.add_message(message_proto); + + // Return MESSAGE type with the type name + Ok(( + prost_types::field_descriptor_proto::Type::Message, + Some(message_name), + )) + } + + ComplexType::Array(element_type) => { + // Arrays become repeated fields + // The element type determines the field type + match element_type.as_ref() { + ComplexType::Primitive(primitive) => { + let proto_type = map_primitive_to_protobuf(primitive); + Ok((proto_type, None)) + } + ComplexType::Struct(_) => { + // Array of structs - need to generate the struct message + let element_message_name = + format!("{}_element", sanitize_message_name(path_prefix)); + let (_, type_name) = map_complex_type_to_protobuf( + element_type, + &element_message_name, + collector, + )?; + + Ok(( + prost_types::field_descriptor_proto::Type::Message, + type_name, + )) + } + ComplexType::Array(_) => { + // Nested arrays not supported by Protobuf directly + Err(ZerobusSinkError::ConfigError { + message: format!("Nested arrays not supported for field: {}", path_prefix), + }) + } + ComplexType::Map { .. } => { + // Array of maps - not directly supported + Err(ZerobusSinkError::ConfigError { + message: format!("Array of maps not supported for field: {}", path_prefix), + }) + } + } + } + + ComplexType::Map { + key_type, + value_type, + } => { + // Protobuf maps are represented as: + // message MapFieldEntry { K key = 1; V value = 2; } + // repeated MapFieldEntry map_field = N; + + // Protobuf maps support any scalar primitive key (int32, int64, bool, string, etc.) + // but not complex types (struct, array, nested map). + let key_primitive = match key_type.as_ref() { + ComplexType::Primitive(p) => p, + _ => { + return Err(ZerobusSinkError::ConfigError { + message: format!( + "MAP with non-scalar keys not supported for field '{}'. \ + Protobuf maps require scalar (primitive) keys. Found key type: {:?}", + path_prefix, key_type + ), + }); + } + }; + + // Check if value is a primitive type + match value_type.as_ref() { + ComplexType::Primitive(value_primitive) => { + // Generate a map entry message for this field + let entry_message_name = + format!("{}_entry", sanitize_message_name(path_prefix)); + let entry_message = generate_map_entry_message( + &entry_message_name, + key_primitive, + value_primitive, + )?; + + collector.add_message(entry_message); + + // Return repeated message type + Ok(( + prost_types::field_descriptor_proto::Type::Message, + Some(entry_message_name), + )) + } + ComplexType::Struct(struct_type) => { + // Map with struct values: generate the value struct message, then a + // map-entry message that references it. + let value_message_name = format!("{}Value", sanitize_message_name(path_prefix)); + let value_message = + generate_struct_message(&value_message_name, struct_type, collector)?; + collector.add_message(value_message); + + let entry_message_name = format!("{}Entry", sanitize_message_name(path_prefix)); + let entry_message = generate_map_entry_message_with_message_value( + &entry_message_name, + key_primitive, + &value_message_name, + )?; + collector.add_message(entry_message); + + Ok(( + prost_types::field_descriptor_proto::Type::Message, + Some(entry_message_name), + )) + } + ComplexType::Array(_) | ComplexType::Map { .. } => { + // Map with complex values + Err(ZerobusSinkError::ConfigError { + message: format!( + "MAP with complex values (ARRAY/MAP) not supported for field '{}'. \ + Protobuf maps require simple value types.", + path_prefix + ), + }) + } + } + } + } +} + +/// Generate a protobuf message definition from a StructType +fn generate_struct_message( + message_name: &str, + struct_type: &StructType, + collector: &mut MessageCollector, +) -> Result { + let mut fields = Vec::new(); + + for (index, field) in struct_type.fields.iter().enumerate() { + // Field number starts at 1 + let field_number = (index + 1) as i32; + + // Recursively map the field type + let path = format!("{}_{}", message_name, field.name); + let (field_type, type_name) = + map_complex_type_to_protobuf(&field.field_type, &path, collector)?; + + // Determine if this is a repeated field (for arrays) + let (label, is_repeated) = if matches!(field.field_type, ComplexType::Array(_)) { + ( + prost_types::field_descriptor_proto::Label::Repeated as i32, + true, + ) + } else if field.nullable { + ( + prost_types::field_descriptor_proto::Label::Optional as i32, + false, + ) + } else { + ( + prost_types::field_descriptor_proto::Label::Required as i32, + false, + ) + }; + + fields.push(prost_types::FieldDescriptorProto { + name: Some(field.name.clone()), + number: Some(field_number), + label: Some(label), + r#type: Some(field_type as i32), + type_name, + extendee: None, + default_value: None, + oneof_index: None, + json_name: Some(field.name.clone()), + options: None, + proto3_optional: Some(field.nullable && !is_repeated), + }); + } + + Ok(prost_types::DescriptorProto { + name: Some(message_name.to_string()), + field: fields, + extension: vec![], + nested_type: vec![], + enum_type: vec![], + extension_range: vec![], + oneof_decl: vec![], + options: None, + reserved_range: vec![], + reserved_name: vec![], + }) +} + +/// Generate a map entry message for protobuf map representation +/// Maps in protobuf are represented as: repeated MapEntry where MapEntry { key, value } +fn generate_map_entry_message( + message_name: &str, + key_type: &PrimitiveType, + value_type: &PrimitiveType, +) -> Result { + let key_proto_type = map_primitive_to_protobuf(key_type); + let value_proto_type = map_primitive_to_protobuf(value_type); + + let fields = vec![ + // key field — any scalar primitive type supported by protobuf maps + prost_types::FieldDescriptorProto { + name: Some("key".to_string()), + number: Some(1), + label: Some(prost_types::field_descriptor_proto::Label::Optional as i32), + r#type: Some(key_proto_type as i32), + type_name: None, + extendee: None, + default_value: None, + oneof_index: None, + json_name: Some("key".to_string()), + options: None, + proto3_optional: Some(false), + }, + // value field + prost_types::FieldDescriptorProto { + name: Some("value".to_string()), + number: Some(2), + label: Some(prost_types::field_descriptor_proto::Label::Optional as i32), + r#type: Some(value_proto_type as i32), + type_name: None, + extendee: None, + default_value: None, + oneof_index: None, + json_name: Some("value".to_string()), + options: None, + proto3_optional: Some(true), + }, + ]; + + Ok(prost_types::DescriptorProto { + name: Some(message_name.to_string()), + field: fields, + extension: vec![], + nested_type: vec![], + enum_type: vec![], + extension_range: vec![], + oneof_decl: vec![], + options: Some(prost_types::MessageOptions { + map_entry: Some(true), // Mark this as a map entry + ..Default::default() + }), + reserved_range: vec![], + reserved_name: vec![], + }) +} + +/// Generate a map entry message where the value is a message (struct) type +fn generate_map_entry_message_with_message_value( + message_name: &str, + key_type: &PrimitiveType, + value_type_name: &str, +) -> Result { + let key_proto_type = map_primitive_to_protobuf(key_type); + + let fields = vec![ + prost_types::FieldDescriptorProto { + name: Some("key".to_string()), + number: Some(1), + label: Some(prost_types::field_descriptor_proto::Label::Optional as i32), + r#type: Some(key_proto_type as i32), + type_name: None, + extendee: None, + default_value: None, + oneof_index: None, + json_name: Some("key".to_string()), + options: None, + proto3_optional: Some(false), + }, + prost_types::FieldDescriptorProto { + name: Some("value".to_string()), + number: Some(2), + label: Some(prost_types::field_descriptor_proto::Label::Optional as i32), + r#type: Some(prost_types::field_descriptor_proto::Type::Message as i32), + type_name: Some(value_type_name.to_string()), + extendee: None, + default_value: None, + oneof_index: None, + json_name: Some("value".to_string()), + options: None, + proto3_optional: Some(true), + }, + ]; + + Ok(prost_types::DescriptorProto { + name: Some(message_name.to_string()), + field: fields, + extension: vec![], + nested_type: vec![], + enum_type: vec![], + extension_range: vec![], + oneof_decl: vec![], + options: Some(prost_types::MessageOptions { + map_entry: Some(true), + ..Default::default() + }), + reserved_range: vec![], + reserved_name: vec![], + }) +} + +// The function converts Unity Catalog field names into valid protobuf message type names: +// When generating protobuf descriptors from Unity Catalog schemas, nested structures (structs, arrays of structs, maps) +// need to become protobuf message types. Protobuf message names must: +// 1. Start with a letter (not _ or digit) +// 2. Be alphanumeric (no special characters) +// 3. Follow PascalCase convention +fn sanitize_message_name(name: &str) -> String { + // Convert to PascalCase and remove invalid characters + let mut result = String::new(); + let mut capitalize_next = true; + + for c in name.chars() { + if c.is_alphanumeric() { + if capitalize_next { + result.push(c.to_ascii_uppercase()); + capitalize_next = false; + } else { + result.push(c); + } + } else { + capitalize_next = true; + } + } + + // Ensure it starts with a letter + if result.is_empty() || !result.chars().next().unwrap().is_alphabetic() { + result.insert(0, 'M'); + } + + result +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_map_simple_types() { + let test_cases = vec![ + ("STRING", prost_types::field_descriptor_proto::Type::String), + ("INT", prost_types::field_descriptor_proto::Type::Int32), + ("BIGINT", prost_types::field_descriptor_proto::Type::Int64), + ("BOOLEAN", prost_types::field_descriptor_proto::Type::Bool), + ("DOUBLE", prost_types::field_descriptor_proto::Type::Double), + ( + "TIMESTAMP", + prost_types::field_descriptor_proto::Type::Int64, // Unix timestamp in microseconds + ), + ("BINARY", prost_types::field_descriptor_proto::Type::Bytes), + ]; + + for (databricks_type, expected_proto_type) in test_cases { + let result = map_simple_databricks_type(databricks_type); + assert!(result.is_ok()); + assert_eq!(result.unwrap(), expected_proto_type); + } + } + + #[test] + fn test_generate_descriptor_simple_schema() { + let schema = UnityCatalogTableSchema { + name: "test_table".to_string(), + catalog_name: "test_catalog".to_string(), + schema_name: "test_schema".to_string(), + columns: vec![ + UnityCatalogColumn { + name: "id".to_string(), + type_text: "bigint".to_string(), + type_name: "BIGINT".to_string(), + position: 1, + nullable: false, + type_json: "{}".to_string(), + }, + UnityCatalogColumn { + name: "message".to_string(), + type_text: "string".to_string(), + type_name: "STRING".to_string(), + position: 2, + nullable: true, + type_json: "{}".to_string(), + }, + ], + }; + + let result = generate_descriptor_from_schema(&schema); + assert!(result.is_ok()); + + let descriptor = result.unwrap(); + assert_eq!(descriptor.fields().len(), 2); + + let id_field = descriptor.get_field_by_name("id"); + assert!(id_field.is_some()); + + let message_field = descriptor.get_field_by_name("message"); + assert!(message_field.is_some()); + } + + #[test] + fn test_parse_struct_type_json() { + let type_json = r#"{ + "type": "struct", + "fields": [ + { + "name": "job_id", + "type": "long", + "nullable": true + }, + { + "name": "task_run_id", + "type": "long", + "nullable": true + } + ] + }"#; + + let result = parse_type_json(type_json); + assert!(result.is_ok()); + + match result.unwrap() { + ComplexType::Struct(struct_type) => { + assert_eq!(struct_type.fields.len(), 2); + assert_eq!(struct_type.fields[0].name, "job_id"); + assert_eq!(struct_type.fields[1].name, "task_run_id"); + } + _ => panic!("Expected struct type"), + } + } + + #[test] + fn test_parse_array_type_json() { + let type_json = r#"{ + "type": "array", + "elementType": "string" + }"#; + + let result = parse_type_json(type_json); + assert!(result.is_ok()); + + match result.unwrap() { + ComplexType::Array(element_type) => match element_type.as_ref() { + ComplexType::Primitive(PrimitiveType::String) => {} + _ => panic!("Expected string element type"), + }, + _ => panic!("Expected array type"), + } + } + + #[test] + fn test_parse_map_type_json() { + let type_json = r#"{ + "type": "map", + "keyType": "string", + "valueType": "string" + }"#; + + let result = parse_type_json(type_json); + assert!(result.is_ok()); + + match result.unwrap() { + ComplexType::Map { + key_type, + value_type, + } => { + assert!(matches!( + key_type.as_ref(), + ComplexType::Primitive(PrimitiveType::String) + )); + assert!(matches!( + value_type.as_ref(), + ComplexType::Primitive(PrimitiveType::String) + )); + } + _ => panic!("Expected map type"), + } + } + + #[test] + fn test_generate_descriptor_with_map() { + let schema = UnityCatalogTableSchema { + name: "test_table".to_string(), + catalog_name: "test_catalog".to_string(), + schema_name: "test_schema".to_string(), + columns: vec![ + UnityCatalogColumn { + name: "id".to_string(), + type_text: "bigint".to_string(), + type_name: "BIGINT".to_string(), + position: 1, + nullable: false, + type_json: "{}".to_string(), + }, + UnityCatalogColumn { + name: "attributes".to_string(), + type_text: "map".to_string(), + type_name: "MAP".to_string(), + position: 2, + nullable: true, + type_json: r#"{"type":"map","keyType":"string","valueType":"string"}"# + .to_string(), + }, + ], + }; + + let result = generate_descriptor_from_schema(&schema); + assert!( + result.is_ok(), + "Failed to generate descriptor with MAP type: {:?}", + result.err() + ); + + let descriptor = result.unwrap(); + assert_eq!(descriptor.fields().len(), 2); + + let id_field = descriptor.get_field_by_name("id"); + assert!(id_field.is_some()); + + let attributes_field = descriptor.get_field_by_name("attributes"); + assert!(attributes_field.is_some()); + } + + #[test] + fn test_strict_validation_fails_on_unsupported() { + let schema = UnityCatalogTableSchema { + name: "test_table".to_string(), + catalog_name: "test_catalog".to_string(), + schema_name: "test_schema".to_string(), + columns: vec![UnityCatalogColumn { + name: "unsupported_col".to_string(), + type_text: "struct<...>".to_string(), + type_name: "STRUCT".to_string(), + position: 1, + nullable: true, + type_json: "invalid json".to_string(), // Malformed type_json + }], + }; + + let result = generate_descriptor_from_schema(&schema); + assert!(result.is_err(), "Expected error for malformed type_json"); + } + + // ========== Fixture-based Tests ========== + // These tests use real Unity Catalog schema responses saved as JSON fixtures + + #[test] + fn test_fixture_simple_table() { + let json = include_str!("tests/fixtures/simple_table_schema.json"); + let schema: UnityCatalogTableSchema = + serde_json::from_str(json).expect("Failed to parse simple_table fixture"); + + let result = generate_descriptor_from_schema(&schema); + assert!( + result.is_ok(), + "Failed to generate descriptor: {:?}", + result.err() + ); + + let descriptor = result.unwrap(); + assert_eq!(descriptor.fields().len(), 4); + + // Verify all fields exist + assert!(descriptor.get_field_by_name("id").is_some()); + assert!(descriptor.get_field_by_name("name").is_some()); + assert!(descriptor.get_field_by_name("created_at").is_some()); + assert!(descriptor.get_field_by_name("is_active").is_some()); + } + + #[test] + fn test_fixture_all_primitive_types() { + let json = include_str!("tests/fixtures/all_primitive_types_schema.json"); + let schema: UnityCatalogTableSchema = + serde_json::from_str(json).expect("Failed to parse all_primitive_types fixture"); + + let result = generate_descriptor_from_schema(&schema); + assert!( + result.is_ok(), + "Failed to generate descriptor: {:?}", + result.err() + ); + + let descriptor = result.unwrap(); + assert_eq!( + descriptor.fields().len(), + 9, + "Should have 9 primitive type columns" + ); + + // Verify specific types + assert!(descriptor.get_field_by_name("col_string").is_some()); + assert!(descriptor.get_field_by_name("col_int").is_some()); + assert!(descriptor.get_field_by_name("col_long").is_some()); + assert!(descriptor.get_field_by_name("col_double").is_some()); + assert!(descriptor.get_field_by_name("col_float").is_some()); + assert!(descriptor.get_field_by_name("col_boolean").is_some()); + assert!(descriptor.get_field_by_name("col_binary").is_some()); + assert!(descriptor.get_field_by_name("col_timestamp").is_some()); + assert!(descriptor.get_field_by_name("col_date").is_some()); + } + + #[test] + fn test_fixture_nested_struct() { + let json = include_str!("tests/fixtures/nested_struct_schema.json"); + let schema: UnityCatalogTableSchema = + serde_json::from_str(json).expect("Failed to parse nested_struct fixture"); + + let result = generate_descriptor_from_schema(&schema); + assert!( + result.is_ok(), + "Failed to generate descriptor for nested struct: {:?}", + result.err() + ); + + let descriptor = result.unwrap(); + assert_eq!(descriptor.fields().len(), 1); + + // Verify nested struct field exists + let user_info_field = descriptor.get_field_by_name("user_info"); + assert!(user_info_field.is_some(), "user_info field should exist"); + } + + #[test] + fn test_fixture_array_of_structs() { + let json = include_str!("tests/fixtures/array_of_structs_schema.json"); + let schema: UnityCatalogTableSchema = + serde_json::from_str(json).expect("Failed to parse array_of_structs fixture"); + + let result = generate_descriptor_from_schema(&schema); + assert!( + result.is_ok(), + "Failed to generate descriptor for array of structs: {:?}", + result.err() + ); + + let descriptor = result.unwrap(); + assert_eq!(descriptor.fields().len(), 1); + + // Verify array field exists + let transactions_field = descriptor.get_field_by_name("transactions"); + assert!( + transactions_field.is_some(), + "transactions field should exist" + ); + } + + #[test] + fn test_fixture_mixed_types() { + // Tests a schema with 5 columns covering: timestamp, bigint, nested struct, + // array, and map. + let json = include_str!("tests/fixtures/mixed_types_schema.json"); + let schema: UnityCatalogTableSchema = + serde_json::from_str(json).expect("Failed to parse mixed_types fixture"); + + let result = generate_descriptor_from_schema(&schema); + assert!( + result.is_ok(), + "Failed to generate descriptor for mixed_types: {:?}", + result.err() + ); + + let descriptor = result.unwrap(); + assert_eq!(descriptor.fields().len(), 5, "Should have 5 columns"); + + // Verify all fields exist: timestamp, bigint, struct, array, map + assert!( + descriptor.get_field_by_name("field_001").is_some(), + "Should have field_001 (timestamp)" + ); + assert!( + descriptor.get_field_by_name("field_002").is_some(), + "Should have field_002 (bigint)" + ); + assert!( + descriptor.get_field_by_name("field_003").is_some(), + "Should have field_003 (STRUCT)" + ); + assert!( + descriptor.get_field_by_name("field_007").is_some(), + "Should have field_007 (ARRAY)" + ); + assert!( + descriptor.get_field_by_name("field_008").is_some(), + "Should have field_008 (MAP)" + ); + } + + #[test] + fn test_fixture_mixed_types_struct_parsing() { + // Test that the nested STRUCT column is properly parsed + let json = include_str!("tests/fixtures/mixed_types_schema.json"); + let schema: UnityCatalogTableSchema = + serde_json::from_str(json).expect("Failed to parse mixed_types fixture"); + + // Find the struct column (field_003) + let struct_col = schema + .columns + .iter() + .find(|c| c.name == "field_003") + .expect("Should have field_003 column"); + + // Parse its type_json + let result = parse_type_json(&struct_col.type_json); + assert!(result.is_ok(), "Should parse field_003 type_json"); + + match result.unwrap() { + ComplexType::Struct(struct_type) => { + assert_eq!(struct_type.fields.len(), 1, "Should have 1 field"); + assert_eq!(struct_type.fields[0].name, "field_004"); + + // Verify nested struct + match &struct_type.fields[0].field_type { + ComplexType::Struct(nested) => { + assert_eq!(nested.fields.len(), 2, "Nested struct should have 2 fields"); + assert_eq!(nested.fields[0].name, "field_005"); + assert_eq!(nested.fields[1].name, "field_006"); + } + _ => panic!("Expected nested struct"), + } + } + _ => panic!("Expected struct type for field_003"), + } + } + + #[test] + fn test_fixture_error_handling_empty_type_json() { + // Test that empty type_json is handled correctly + let result = parse_type_json(""); + assert!(result.is_err(), "Should fail on empty type_json"); + + let result = parse_type_json("{}"); + assert!(result.is_err(), "Should fail on empty object type_json"); + } + + #[test] + fn test_fixture_error_handling_invalid_json() { + // Test that invalid JSON is handled correctly + let result = parse_type_json("not valid json"); + assert!(result.is_err(), "Should fail on invalid JSON"); + } + + #[test] + fn test_fixture_error_handling_missing_type_field() { + // Test that missing 'type' field is handled + let result = parse_type_json(r#"{"fields": []}"#); + assert!(result.is_err(), "Should fail when 'type' field is missing"); + } + + // ========== Comprehensive Proto Compatibility Test ========== + + /// Helper function to assert a field exists with expected type + fn assert_field_exists_with_type( + descriptor: &prost_reflect::MessageDescriptor, + field_name: &str, + expected_type: &str, + ) { + let field = descriptor + .get_field_by_name(field_name) + .unwrap_or_else(|| panic!("Field '{}' should exist", field_name)); + + let actual_type = format_field_type_simple(&field); + assert_eq!( + actual_type, expected_type, + "Field '{}' should have type '{}'", + field_name, expected_type + ); + } + + /// Helper function to assert a field is a message type with expected name + fn assert_field_is_message( + descriptor: &prost_reflect::MessageDescriptor, + field_name: &str, + message_type: &str, + ) { + let field = descriptor + .get_field_by_name(field_name) + .unwrap_or_else(|| panic!("Field '{}' should exist", field_name)); + + match field.kind() { + prost_reflect::Kind::Message(msg) => { + assert_eq!( + msg.name(), + message_type, + "Field '{}' should be message type '{}'", + field_name, + message_type + ); + } + _ => panic!("Field '{}' should be a message type", field_name), + } + } + + /// Helper function to assert a field is repeated (array) + fn assert_field_is_repeated(descriptor: &prost_reflect::MessageDescriptor, field_name: &str) { + let field = descriptor + .get_field_by_name(field_name) + .unwrap_or_else(|| panic!("Field '{}' should exist", field_name)); + + assert!( + field.is_list(), + "Field '{}' should be repeated/list", + field_name + ); + } + + /// Helper function to format field type as simple string + fn format_field_type_simple(field: &prost_reflect::FieldDescriptor) -> String { + use prost_reflect::Kind; + match field.kind() { + Kind::String => "string".to_string(), + Kind::Int64 => "int64".to_string(), + Kind::Int32 => "int32".to_string(), + Kind::Bool => "bool".to_string(), + Kind::Double => "double".to_string(), + Kind::Float => "float".to_string(), + Kind::Bytes => "bytes".to_string(), + Kind::Message(msg) => msg.name().to_string(), + _ => "unknown".to_string(), + } + } + + #[test] + fn test_nested_structs_complete_schema() { + // Verifies that a 15-column schema with nested structs, arrays, and + // various primitive types generates a correct protobuf descriptor. + + let json = include_str!("tests/fixtures/nested_structs_complete_schema.json"); + let schema: UnityCatalogTableSchema = + serde_json::from_str(json).expect("Failed to parse nested_structs_complete schema"); + + let descriptor = generate_descriptor_from_schema(&schema) + .expect("Failed to generate descriptor from complete schema"); + + // === 1. VERIFY MAIN MESSAGE STRUCTURE === + + assert_eq!( + descriptor.name(), + "test_schema_nested_structs_table", + "Main message should have expected name" + ); + + assert_eq!( + descriptor.fields().len(), + 15, + "Should have exactly 15 fields" + ); + + // === 2. VERIFY PRIMITIVE FIELDS === + + assert_field_exists_with_type(&descriptor, "field_001", "int64"); // bigint + assert_field_exists_with_type(&descriptor, "field_002", "string"); + assert_field_exists_with_type(&descriptor, "field_003", "string"); + assert_field_exists_with_type(&descriptor, "field_004", "string"); + assert_field_exists_with_type(&descriptor, "field_005", "string"); + assert_field_exists_with_type(&descriptor, "field_006", "int64"); // bigint + assert_field_exists_with_type(&descriptor, "field_007", "int64"); // bigint + assert_field_exists_with_type(&descriptor, "field_028", "bool"); + assert_field_exists_with_type(&descriptor, "field_029", "string"); + assert_field_exists_with_type(&descriptor, "field_030", "string"); + assert_field_exists_with_type(&descriptor, "field_031", "bool"); + + // === 3. VERIFY COMPLEX MESSAGE TYPES === + + assert_field_is_message(&descriptor, "field_008", "Field008"); + assert_field_is_message(&descriptor, "field_018", "Field018"); + assert_field_is_message(&descriptor, "field_021", "Field021"); + + // === 4. VERIFY ARRAY FIELD === + + assert_field_is_repeated(&descriptor, "field_027"); + + let array_field = descriptor + .get_field_by_name("field_027") + .expect("field_027 should exist"); + let element_type = format_field_type_simple(&array_field); + assert_eq!(element_type, "int64", "field_027 should be repeated int64"); + + // === 5. VERIFY NESTED MESSAGE: Field018 (2 string fields) === + + let field_018 = descriptor + .get_field_by_name("field_018") + .expect("field_018 should exist"); + + if let prost_reflect::Kind::Message(msg) = field_018.kind() { + assert_eq!(msg.fields().len(), 2, "Field018 should have 2 fields"); + assert!(msg.get_field_by_name("field_019").is_some()); + assert!(msg.get_field_by_name("field_020").is_some()); + } else { + panic!("field_018 should be a message type"); + } + + // === 6. VERIFY NESTED MESSAGE: Field021 (6 fields incl. int32) === + + let field_021 = descriptor + .get_field_by_name("field_021") + .expect("field_021 should exist"); + + if let prost_reflect::Kind::Message(msg) = field_021.kind() { + assert_eq!(msg.fields().len(), 6, "Field021 should have 6 fields"); + + let expected_fields = vec![ + "field_022", + "field_023", + "field_020", + "field_024", + "field_025", + "field_026", + ]; + for field_name in expected_fields { + assert!( + msg.get_field_by_name(field_name).is_some(), + "Field021 should have {} field", + field_name + ); + } + + // Verify field_026 is int32 + let f026 = msg + .get_field_by_name("field_026") + .expect("field_026 should exist"); + match f026.kind() { + prost_reflect::Kind::Int32 => {} + _ => panic!("field_026 should be int32"), + } + } else { + panic!("field_021 should be a message type"); + } + + // === 7. VERIFY NESTED MESSAGE: Field008 (4 nested struct fields) === + + let field_008 = descriptor + .get_field_by_name("field_008") + .expect("field_008 should exist"); + + if let prost_reflect::Kind::Message(msg) = field_008.kind() { + let expected_nested = vec!["field_009", "field_012", "field_014", "field_016"]; + for nested in expected_nested { + assert!( + msg.get_field_by_name(nested).is_some(), + "Field008 should have {} field", + nested + ); + } + + // Verify field_009 nested structure has 2 fields + let f009 = msg + .get_field_by_name("field_009") + .expect("field_009 should exist"); + if let prost_reflect::Kind::Message(nested_msg) = f009.kind() { + assert!(nested_msg.get_field_by_name("field_010").is_some()); + assert!(nested_msg.get_field_by_name("field_011").is_some()); + } + } else { + panic!("field_008 should be a message type"); + } + } + + #[test] + fn test_proto_schema_snapshot() { + // Snapshot test: verify the generated proto text matches expected format + let json = include_str!("tests/fixtures/nested_structs_complete_schema.json"); + let schema: UnityCatalogTableSchema = + serde_json::from_str(json).expect("Failed to parse nested_structs_complete schema"); + + let descriptor = + generate_descriptor_from_schema(&schema).expect("Failed to generate descriptor"); + + // Format as proto text + let proto_text = format_descriptor_as_proto(&descriptor); + + // Verify key structures are present in the proto text + assert!( + proto_text.contains("message test_schema_nested_structs_table"), + "Proto should have main message definition" + ); + assert!( + proto_text.contains("string field_003"), + "Proto should have field_003 (string)" + ); + assert!( + proto_text.contains("int64 field_007"), + "Proto should have field_007 (int64)" + ); + assert!( + proto_text.contains("repeated int64 field_027"), + "Proto should have field_027 as repeated int64" + ); + assert!( + proto_text.contains("message Field018"), + "Proto should have Field018 nested message" + ); + assert!( + proto_text.contains("message Field021"), + "Proto should have Field021 nested message" + ); + assert!( + proto_text.contains("message Field008"), + "Proto should have Field008 nested message" + ); + } + + #[test] + fn test_complex_schema_with_all_type_patterns() { + // Regression test: verifies that a large 91-column schema exercising + // all supported Unity Catalog type patterns generates a valid protobuf + // descriptor without errors. + // + // Coverage: + // - map (field_142) — non-string scalar key + // - map (field_143) — non-string scalar key + // - map (field_725) — MAP with struct value + // - ARRAY with deeply nested fields + // - Deeply nested STRUCTs (4+ levels) + + let json = include_str!("tests/fixtures/complex_nested_types_schema.json"); + let schema: UnityCatalogTableSchema = + serde_json::from_str(json).expect("Failed to parse schema fixture"); + + assert_eq!( + schema.columns.len(), + 91, + "Fixture must contain all 91 columns" + ); + + let descriptor = generate_descriptor_from_schema(&schema) + .expect("Should succeed: all column types must be supported"); + + let proto_text = format_descriptor_as_proto(&descriptor); + + // --- non-string scalar map keys --- + assert!( + proto_text.contains("field_142"), + "Should contain field_142 (map)" + ); + assert!( + proto_text.contains("field_143"), + "Should contain field_143 (map)" + ); + + // --- MAP with STRUCT value --- + assert!( + proto_text.contains("field_725"), + "Should contain field_725 (map)" + ); + + // --- key top-level fields --- + assert!( + descriptor.get_field_by_name("field_001").is_some(), + "Should have field_001 (timestamp)" + ); + assert!( + descriptor.get_field_by_name("field_002").is_some(), + "Should have field_002 (string)" + ); + assert!( + descriptor.get_field_by_name("field_007").is_some(), + "Should have field_007 (deeply nested metadata struct)" + ); + assert!( + descriptor.get_field_by_name("field_117").is_some(), + "Should have field_117 (struct with map fields)" + ); + assert!( + descriptor.get_field_by_name("field_721").is_some(), + "Should have field_721 (struct containing MAP)" + ); + assert!( + descriptor.get_field_by_name("field_149").is_some(), + "Should have field_149 (ARRAY)" + ); + assert!( + descriptor.get_field_by_name("field_278").is_some(), + "Should have field_278 (deeply nested ARRAY)" + ); + + // total field count must match all 91 columns + assert_eq!( + descriptor.fields().len(), + 91, + "Descriptor should have exactly 91 fields" + ); + } +} diff --git a/src/sinks/mod.rs b/src/sinks/mod.rs index 3a9c3c049a0fa..fb8621491081c 100644 --- a/src/sinks/mod.rs +++ b/src/sinks/mod.rs @@ -40,6 +40,8 @@ pub mod clickhouse; pub mod console; #[cfg(feature = "sinks-databend")] pub mod databend; +#[cfg(feature = "sinks-databricks-zerobus")] +pub mod databricks_zerobus; #[cfg(any( feature = "sinks-datadog_events", feature = "sinks-datadog_logs", diff --git a/src/sinks/util/encoding.rs b/src/sinks/util/encoding.rs index 4cc49b00f358b..d615b2eb69ff5 100644 --- a/src/sinks/util/encoding.rs +++ b/src/sinks/util/encoding.rs @@ -99,7 +99,6 @@ impl Encoder for (Transformer, vector_lib::codecs::Encoder<()>) { } } -#[cfg(feature = "codecs-arrow")] impl Encoder> for (Transformer, vector_lib::codecs::BatchEncoder) { fn encode_input( &self, @@ -140,7 +139,6 @@ impl Encoder> for (Transformer, vector_lib::codecs::EncoderKind) { vector_lib::codecs::EncoderKind::Framed(encoder) => { (self.0.clone(), *encoder.clone()).encode_input(events, writer) } - #[cfg(feature = "codecs-arrow")] vector_lib::codecs::EncoderKind::Batch(encoder) => { (self.0.clone(), encoder.clone()).encode_input(events, writer) } diff --git a/website/cue/reference/components/sinks/databricks_zerobus.cue b/website/cue/reference/components/sinks/databricks_zerobus.cue new file mode 100644 index 0000000000000..725233d17be29 --- /dev/null +++ b/website/cue/reference/components/sinks/databricks_zerobus.cue @@ -0,0 +1,131 @@ +package metadata + +components: sinks: databricks_zerobus: { + title: "Databricks Zerobus" + + classes: { + commonly_used: false + delivery: "at_least_once" + development: "beta" + egress_method: "batch" + service_providers: ["Databricks"] + stateful: false + } + + features: { + auto_generated: true + acknowledgements: true + healthcheck: enabled: true + send: { + batch: { + enabled: true + common: false + max_bytes: 10_000_000 + timeout_secs: 300.0 + } + compression: enabled: false + encoding: enabled: false + proxy: enabled: false + request: { + enabled: true + headers: false + } + tls: enabled: false + to: { + service: services.databricks_zerobus + + interface: { + socket: { + direction: "outgoing" + protocols: ["https"] + ssl: "required" + } + } + } + } + } + + support: { + requirements: [ + """ + A [Databricks](\(urls.databricks)) workspace with [Unity Catalog](\(urls.databricks_unity_catalog)) enabled. + """, + """ + OAuth 2.0 client credentials (client ID and client secret) with permissions to write to the target table. + """, + ] + warnings: [] + notices: [] + } + + configuration: generated.components.sinks.databricks_zerobus.configuration + + input: { + logs: true + metrics: null + traces: false + } + + how_it_works: { + authentication: { + title: "Authentication" + body: """ + The Databricks Zerobus sink authenticates using OAuth 2.0 client credentials. + You must provide a `client_id` and `client_secret` that have been granted + permissions to write to the target Unity Catalog table. + """ + } + + schema: { + title: "Schema" + body: """ + The sink requires a schema to encode events into protobuf format. The schema can + be provided in two ways: + + #### Unity Catalog (default) + + When `schema.type` is set to `unity_catalog`, the sink automatically fetches the + table schema from the Unity Catalog API at startup. This is the recommended approach + as it ensures the schema always matches the target table. + + ```yaml + sinks: + zerobus: + type: databricks_zerobus + schema: + type: unity_catalog + ``` + + #### Protobuf descriptor file + + You can provide a pre-compiled protobuf descriptor file. This is useful for + development or when the Unity Catalog API is not accessible. + + ```yaml + sinks: + zerobus: + type: databricks_zerobus + schema: + type: path + path: /path/to/schema.desc + message_type: package.MessageName + ``` + + Descriptor files can be generated using protoc: + + ```sh + protoc --descriptor_set_out=schema.desc --include_imports your_schema.proto + ``` + """ + } + + batching: { + title: "Batching" + body: """ + Events are batched before being sent to Zerobus. Each event is individually + serialized as a protobuf message, and the batch is sent as a single request. + The maximum batch size is 10MB, which is enforced by the Zerobus SDK. + """ + } + } +} diff --git a/website/cue/reference/services/databricks_zerobus.cue b/website/cue/reference/services/databricks_zerobus.cue new file mode 100644 index 0000000000000..1091c215a5d68 --- /dev/null +++ b/website/cue/reference/services/databricks_zerobus.cue @@ -0,0 +1,10 @@ +package metadata + +services: databricks_zerobus: { + name: "Databricks Zerobus" + thing: "a \(name) ingestion stream" + url: urls.databricks + versions: null + + description: "[Databricks](\(urls.databricks)) is a unified analytics platform. The Zerobus sink streams observability data to Databricks Unity Catalog tables via the Zerobus ingestion service, using protobuf encoding for efficient data transfer." +} diff --git a/website/cue/reference/urls.cue b/website/cue/reference/urls.cue index 239599effed75..0133e581f82e5 100644 --- a/website/cue/reference/urls.cue +++ b/website/cue/reference/urls.cue @@ -124,6 +124,8 @@ urls: { cue: "https://cuelang.org/" csv: "\(wikipedia)/wiki/Comma-separated_values" dag: "\(wikipedia)/wiki/Directed_acyclic_graph" + databricks: "https://www.databricks.com" + databricks_unity_catalog: "https://docs.databricks.com/en/data-governance/unity-catalog/index.html" databend: "https://databend.rs" databend_rest: "https://databend.rs/doc/integrations/api/rest" databend_cloud: "https://www.databend.com"