Skip to content

Commit

Permalink
Included an unshaded guava specifically for BigQuery usage. (#177)
Browse files Browse the repository at this point in the history
Note that the guava versions used by Flink and BigQuery can be
incompatible with each other. The final uber jar of a consumer should
include both shaded versions so that Flink and BigQuery connectors can
function correctly with their own guava dependency. Without this, Maven
will resolve to a single version of guava randomly picked from some
transitive dependency and may result in runtime exceptions of a Flink
job.
  • Loading branch information
nika-qubit authored Oct 29, 2024
1 parent 14263b7 commit e9ca296
Show file tree
Hide file tree
Showing 2 changed files with 57 additions and 25 deletions.
7 changes: 7 additions & 0 deletions flink-connector-bigquery-common/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,13 @@ under the License.
<artifactId>google-cloud-bigquery</artifactId>
</dependency>

<!-- Not directly consumed, kept for shading -->
<!-- This will be a different version of guava than the one used by Flink -->
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>

<!-- org.apache.avro - used by SchemaTransform -->
<dependency>
<groupId>org.apache.avro</groupId>
Expand Down
75 changes: 50 additions & 25 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ under the License.
<properties>
<revision>0.4-SNAPSHOT</revision>
<flink.version>1.17.1</flink.version>
<bigquery.guava.version>32.1.3-jre</bigquery.guava.version>
<google-lib-bom.version>26.33.0</google-lib-bom.version>

<junit5.version>5.9.3</junit5.version>
Expand Down Expand Up @@ -175,6 +176,13 @@ under the License.
<scope>import</scope>
</dependency>

<!-- The guava version used by BigQuery, not Flink -->
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${bigquery.guava.version}</version>
</dependency>

<!-- Utilities -->
<dependency>
<groupId>dev.failsafe</groupId>
Expand Down Expand Up @@ -347,32 +355,49 @@ under the License.
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<executions>
<execution>
<id>shade-flink</id>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<artifactSet>
<includes>
<include>com.google.guava:guava:*</include>
</includes>
</artifactSet>
</configuration>
</execution>
</executions>
</plugin>

<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>flatten-maven-plugin</artifactId>
<version>1.5.0</version>
<configuration>
<updatePomFile>true</updatePomFile>
<flattenMode>resolveCiFriendliesOnly</flattenMode>
</configuration>
<executions>
<execution>
<id>flatten</id>
<phase>verify</phase>
<goals>
<goal>flatten</goal>
</goals>
</execution>
<execution>
<id>flatten.clean</id>
<phase>clean</phase>
<goals>
<goal>clean</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>flatten-maven-plugin</artifactId>
<version>1.5.0</version>
<configuration>
<updatePomFile>true</updatePomFile>
<flattenMode>resolveCiFriendliesOnly</flattenMode>
</configuration>
<executions>
<execution>
<id>flatten</id>
<phase>verify</phase>
<goals>
<goal>flatten</goal>
</goals>
</execution>
<execution>
<id>flatten.clean</id>
<phase>clean</phase>
<goals>
<goal>clean</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.commonjava.maven.plugins</groupId>
<artifactId>directory-maven-plugin</artifactId>
Expand Down

0 comments on commit e9ca296

Please sign in to comment.