Skip to content

Commit

Permalink
Barrage: Version Bumps, Feat Deprecation, New Feat Flags (#41)
Browse files Browse the repository at this point in the history 
- version bump barrage artifacts for next release to be 0.7.0
- version bump to Arrow 18, and similar requirements for gRPC / flatbuffer dependencies
- mark ColumnConversionModes as deprecated
- remove session management types and enum values; these were never used
- added preview_list_length_limit which enables the server to truncate lists/arrays at any depth of table content (offers head / tail of `n` elements)
---------

Co-authored-by: Colin Alworth <colinalworth@deephaven.io>
  • Loading branch information
@nbauernfeind @niloc132
nbauernfeind and niloc132 authored Nov 11, 2024
1 parent 43c12dd commit 36a49ef
Show file tree
Hide file tree
Showing 20 changed files with 1,222 additions and 515 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/package.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ jobs:
- name: Setup JDK
uses: actions/setup-java@v1
with:
java-version: '8.0.282'
java-version: '17.0.11'

- name: Package with Maven
run: mvn --file java/pom.xml --batch-mode package
run: mvn --file java/pom.xml --batch-mode package
71 changes: 32 additions & 39 deletions format/Barrage.fbs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,10 @@ enum BarrageMessageType : byte {
/// if the msg_payload is empty.
None = 0,

/// for session management (not-yet-used)
NewSessionRequest = 1,
RefreshSessionRequest = 2,
SessionInfoResponse = 3,
/// enum values 1 - 3 are reserved for future use
UNUSED_1 = 1,
UNUSED_2 = 2,
UNUSED_3 = 3,

/// for subscription parsing/management (aka DoPut, DoExchange)
BarrageSerializationOptions = 4,
Expand All @@ -47,42 +47,13 @@ table BarrageMessageWrapper {
msg_payload: [byte];
}

/// Establish a new session.
table NewSessionRequest {
/// A nested protocol version (gets delegated to handshake)
protocol_version: uint;

/// Arbitrary auth/handshake info.
payload: [byte];
}

/// Refresh the provided session.
table RefreshSessionRequest {
/// this session token is only required if it is the first request of a handshake rpc stream
session: [byte];
}

/// Information about the current session state.
table SessionInfoResponse {
/// this is the metadata header to identify this session with future requests; it must be lower-case and remain static for the life of the session
metadata_header: [byte];

/// this is the session_token; note that it may rotate
session_token: [byte];

/// a suggested time for the user to refresh the session if they do not do so earlier; value is denoted in milliseconds since epoch
token_refresh_deadline_ms: long;
}

/// There will always be types that cannot be easily supported over IPC. These are the options:
/// Stringify (default) - Pretend the column is a string when sending over Arrow Flight (default)
/// JavaSerialization - Use java serialization; the client is responsible for the deserialization
/// ThrowError - Refuse to send the column and throw an internal error sharing as much detail as possible
/// There will always be types that cannot be easily supported over IPC. While column conversion mode is no longer
/// supported, users can more explicitly configure the encoding/decoding behavior of the server.
enum ColumnConversionMode : byte { Stringify = 1, JavaSerialization, ThrowError }

table BarrageSubscriptionOptions {
/// see enum for details
column_conversion_mode: ColumnConversionMode = Stringify;
/// Column conversion mode is no longer supported.
column_conversion_mode: ColumnConversionMode = Stringify (deprecated);

/// Deephaven reserves a value in the range of primitives as a custom NULL value. This enables more efficient transmission
/// by eliminating the additional complexity of the validity buffer.
Expand All @@ -109,6 +80,15 @@ table BarrageSubscriptionOptions {
/// If true, the server will wrap columns with a list. This is useful for clients that do not support modified batches
/// with columns of differing lengths.
columns_as_list: bool;

/// The maximum length of any list / array to encode.
/// - If zero, list lengths will not be limited.
/// - If non-zero, the server will limit the length of any encoded list / array to the absolute value of the returned length.
/// - If less than zero, the server will encode elements from the end of the list / array, rather than from the beginning.
///
/// Note: The server is unable to indicate when truncation occurs. To detect truncation request one more element than
/// the maximum number you wish to display.
preview_list_length_limit: long = 0;
}

/// Describes the subscription the client would like to acquire.
Expand All @@ -129,11 +109,15 @@ table BarrageSubscriptionRequest {
/// When this is set the viewport RowSet will be inverted against the length of the table. That is to say
/// every position value is converted from `i` to `n - i - 1` if the table has `n` rows.
reverse_viewport: bool;

/// If this is set, the server will parrot this subscription token in the response. This token can be used to identify
/// which subscription the server is now respecting.
subscription_token: [byte];
}

table BarrageSnapshotOptions {
/// see enum for details
column_conversion_mode: ColumnConversionMode = Stringify;
/// Column conversion mode is no longer supported.
column_conversion_mode: ColumnConversionMode = Stringify (deprecated);

/// Deephaven reserves a value in the range of primitives as a custom NULL value. This enables more efficient transmission
/// by eliminating the additional complexity of the validity buffer.
Expand All @@ -148,6 +132,15 @@ table BarrageSnapshotOptions {
/// of one row per batch). If the message size limit cannot be met due to large row sizes, the server will throw a
/// `Status.RESOURCE_EXHAUSTED` exception
max_message_size: int;

/// The maximum length of any list / array to encode.
/// - If zero, list lengths will not be limited.
/// - If non-zero, the server will limit the length of any encoded list / array to the absolute value of the returned length.
/// - If less than zero, the server will encode elements from the end of the list / array, rather than from the beginning.
///
/// Note: The server is unable to indicate when truncation occurs. To detect truncation request one more element than
/// the maximum number you wish to display.
preview_list_length_limit: long = 0;
}

/// Describes the snapshot the client would like to acquire.
Expand Down
2 changes: 1 addition & 1 deletion header
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
Copyright 2020 Deephaven Data Labs
Copyright 2020-2024 Deephaven Data Labs

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
Expand Down
138 changes: 26 additions & 112 deletions java/barrage-core/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,31 +19,31 @@
<parent>
<groupId>io.deephaven.barrage</groupId>
<artifactId>barrage-java-root</artifactId>
<version>0.6.1-SNAPSHOT</version>
<version>0.7.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

<artifactId>barrage-core</artifactId>
<name>Deephaven Barrage Core</name>
<description>An RPC mechanism for transferring ticking Arrow data.</description>
<packaging>jar</packaging>

<properties>
<dep.grpc.version>1.58.0</dep.grpc.version>
<dep.protobuf.version>3.21.9</dep.protobuf.version>
<forkCount>1</forkCount>
<dep.grpc.version>1.65.0</dep.grpc.version>
<dep.protobuf.version>3.25.4</dep.protobuf.version>
<dep.guava.version>33.3.1-jre</dep.guava.version>
<protoOutDir>${project.build.directory}/generated-sources/protobuf</protoOutDir>
</properties>

<dependencies>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-core</artifactId>
<artifactId>grpc-protobuf</artifactId>
<version>${dep.grpc.version}</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-protobuf</artifactId>
<version>${dep.grpc.version}</version>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${dep.guava.version}</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
Expand All @@ -61,15 +61,8 @@
<version>${dep.grpc.version}</version>
</dependency>
<dependency>
<groupId>org.apache.arrow</groupId>
<artifactId>flight-core</artifactId>
</dependency>

<dependency>
<groupId>com.google.api.grpc</groupId>
<artifactId>proto-google-common-protos</artifactId>
<version>1.12.0</version>
<scope>test</scope>
<groupId>org.apache.arrow</groupId>
<artifactId>flight-core</artifactId>
</dependency>
</dependencies>
<build>
Expand All @@ -81,69 +74,6 @@
</extension>
</extensions>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.4.0</version>
<executions>
<execution>
<id>shade-main</id>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<shadedArtifactAttached>true</shadedArtifactAttached>
<shadedClassifierName>shaded</shadedClassifierName>
<artifactSet>
<includes>
<include>io.grpc:*</include>
<include>com.google.protobuf:*</include>
</includes>
</artifactSet>
<relocations>
<relocation>
<pattern>com.google.protobuf</pattern>
<shadedPattern>io.deephaven.barrage.com.google.protobuf</shadedPattern>
</relocation>
</relocations>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
</transformers>
</configuration>
</execution>
<execution>
<id>shade-ext</id>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<shadedArtifactAttached>true</shadedArtifactAttached>
<shadedClassifierName>shaded-ext</shadedClassifierName>
<artifactSet>
<includes>
<include>io.grpc:*</include>
<include>com.google.protobuf:*</include>
</includes>
</artifactSet>
<relocations>
<relocation>
<pattern>com.google.protobuf</pattern>
<shadedPattern>io.deephaven.barrage.com.google.protobuf</shadedPattern>
</relocation>
<relocation>
<pattern>com.google.common</pattern>
<shadedPattern>io.deephaven.barrage.com.google.common</shadedPattern>
</relocation>
</relocations>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
</transformers>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.xolstice.maven.plugins</groupId>
<artifactId>protobuf-maven-plugin</artifactId>
Expand All @@ -159,7 +89,7 @@
<id>src</id>
<configuration>
<protoSourceRoot>${basedir}/../../format/</protoSourceRoot>
<outputDirectory>${project.build.directory}/generated-sources/protobuf</outputDirectory>
<outputDirectory>${protoOutDir}</outputDirectory>
<includes>
<!-- Note we only want to include the BrowserFlight service! -->
<include>**/BrowserFlight.proto</include>
Expand All @@ -172,22 +102,24 @@
</execution>
</executions>
</plugin>
<!-- add the license header to the generated files -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<groupId>com.mycila</groupId>
<artifactId>license-maven-plugin</artifactId>
<version>4.6</version>
<configuration>
<header>${basedir}/../../header</header>
<includes>
<include>**/*.java</include>
</includes>
<defaultBasedir>${protoOutDir}</defaultBasedir>
</configuration>
<executions>
<execution>
<id>analyze</id>
<phase>verify</phase>
<phase>process-sources</phase>
<goals>
<goal>analyze-only</goal>
<goal>format</goal>
</goals>
<configuration>
<ignoredDependencies>
<ignoredDependency>io.grpc:grpc-core:jar:1.30.2</ignoredDependency>
<ignoredDependency>io.grpc:grpc-context:jar:1.30.2</ignoredDependency>
</ignoredDependencies>
</configuration>
</execution>
</executions>
</plugin>
Expand All @@ -204,30 +136,12 @@
</goals>
<configuration>
<sources>
<source>${project.build.directory}/generated-sources/protobuf</source>
<source>${protoOutDir}</source>
</sources>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<version>3.4.2</version>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
29 changes: 29 additions & 0 deletions java/format/README.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
### Updating the flatbuffers generated code

1. Verify that your version of flatc matches the declared dependency:

```bash
$ flatc --version
flatc version 24.3.25
$ grep "dep.fbs.version" java/pom.xml
<dep.fbs.version>24.3.25</dep.fbs.version>
```

2. Generate the flatbuffer java files by performing the following:

```bash
cd $BARRAGE_HOME
# remove the existing files
rm -rf java/format/src
# regenerate from the .fbs files
flatc --java -o java/format/src/main/java format/*.fbs
# generate license headers
mvn compile
```

3. Ensure any new files are added to the git repository:

```bash
cd $BARRAGE_HOME
find java/format/src -type f | while read file; do git add $file; done
```
Loading

0 comments on commit 36a49ef

Please sign in to comment.