Skip to content

Commit

Permalink
Included an unshaded guava specifically for BigQuery usage.
Browse files Browse the repository at this point in the history 
Note that the guava versions used by Flink and BigQuery can be incompatible with each other.
The final uber jar of a consumer should include both versions so that Flink and BigQuery connectors
can function correctly with their own guava dependency. Without this, Maven will resolve to a
single version of guava randomly picked from some transitive dependency and may result in runtime
exceptions of a Flink job.
  • Loading branch information
@nika-qubit
nika-qubit committed Oct 28, 2024
1 parent 14263b7 commit 2b2e100
Show file tree
Hide file tree
Showing 2 changed files with 57 additions and 25 deletions.
7 changes: 7 additions & 0 deletions flink-connector-bigquery-common/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,13 @@ under the License.
<artifactId>google-cloud-bigquery</artifactId>
</dependency>

<!-- Not directly consumed, kept for shading -->
<!-- This will be a different version of guava than the one used by Flink -->
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>

<!-- org.apache.avro - used by SchemaTransform -->
<dependency>
<groupId>org.apache.avro</groupId>
Expand Down
75 changes: 50 additions & 25 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ under the License.
<properties>
<revision>0.4-SNAPSHOT</revision>
<flink.version>1.17.1</flink.version>
<bigquery.guava.version>32.1.3-jre</bigquery.guava.version>
<google-lib-bom.version>26.33.0</google-lib-bom.version>

<junit5.version>5.9.3</junit5.version>
Expand Down Expand Up @@ -175,6 +176,13 @@ under the License.
<scope>import</scope>
</dependency>

<!-- The guava version used by BigQuery, not Flink -->
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${bigquery.guava.version}</version>
</dependency>

<!-- Utilities -->
<dependency>
<groupId>dev.failsafe</groupId>
Expand Down Expand Up @@ -347,32 +355,49 @@ under the License.
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<executions>
<execution>
<id>shade-flink</id>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<artifactSet>
<includes>
<include>com.google.guava:guava:*</include>
</includes>
</artifactSet>
</configuration>
</execution>
</executions>
</plugin>

<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>flatten-maven-plugin</artifactId>
<version>1.5.0</version>
<configuration>
<updatePomFile>true</updatePomFile>
<flattenMode>resolveCiFriendliesOnly</flattenMode>
</configuration>
<executions>
<execution>
<id>flatten</id>
<phase>verify</phase>
<goals>
<goal>flatten</goal>
</goals>
</execution>
<execution>
<id>flatten.clean</id>
<phase>clean</phase>
<goals>
<goal>clean</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>flatten-maven-plugin</artifactId>
<version>1.5.0</version>
<configuration>
<updatePomFile>true</updatePomFile>
<flattenMode>resolveCiFriendliesOnly</flattenMode>
</configuration>
<executions>
<execution>
<id>flatten</id>
<phase>verify</phase>
<goals>
<goal>flatten</goal>
</goals>
</execution>
<execution>
<id>flatten.clean</id>
<phase>clean</phase>
<goals>
<goal>clean</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.commonjava.maven.plugins</groupId>
<artifactId>directory-maven-plugin</artifactId>
Expand Down

0 comments on commit 2b2e100

Please sign in to comment.