Bump to latest Kafka clients 3.7 (#3900)

* Bump to latest Kafka clients Signed-off-by: Pierangelo Di Pilato <pierdipi@redhat.com> * Fix compilation errors Signed-off-by: Pierangelo Di Pilato <pierdipi@redhat.com> --------- Signed-off-by: Pierangelo Di Pilato <pierdipi@redhat.com>
knative-extensions · Jun 6, 2024 · 7d14a4a · 7d14a4a
1 parent 7778ffe
commit 7d14a4a
Show file tree

Hide file tree

Showing 6 changed files with 41 additions and 31 deletions.
diff --git a/data-plane/THIRD-PARTY.txt b/data-plane/THIRD-PARTY.txt
@@ -1,5 +1,5 @@
 
-Lists of 234 third-party dependencies.
+Lists of 245 third-party dependencies.
      (Eclipse Public License - v 1.0) (GNU Lesser General Public License) Logback Classic Module (ch.qos.logback:logback-classic:1.4.14 - http://logback.qos.ch/logback-classic)
      (Eclipse Public License - v 1.0) (GNU Lesser General Public License) Logback Core Module (ch.qos.logback:logback-core:1.4.14 - http://logback.qos.ch/logback-core)
      (Apache License 2.0) brotli4j (com.aayushatharva.brotli4j:brotli4j:1.16.0 - https://github.com/hyperxpro/Brotli4j/brotli4j)
@@ -12,8 +12,9 @@ Lists of 234 third-party dependencies.
      (The Apache Software License, Version 2.0) Jackson-dataformat-YAML (com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.16.1 - https://github.com/FasterXML/jackson-dataformats-text)
      (The Apache Software License, Version 2.0) Jackson datatype: jdk8 (com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.16.1 - https://github.com/FasterXML/jackson-modules-java8/jackson-datatype-jdk8)
      (The Apache Software License, Version 2.0) Jackson datatype: JSR310 (com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.16.1 - https://github.com/FasterXML/jackson-modules-java8/jackson-datatype-jsr310)
-     (The Apache Software License, Version 2.0) jackson-module-scala (com.fasterxml.jackson.module:jackson-module-scala_2.12:2.16.1 - https://github.com/FasterXML/jackson-module-scala)
-     (BSD 2-Clause License) zstd-jni (com.github.luben:zstd-jni:1.5.2-1 - https://github.com/luben/zstd-jni)
+     (The Apache Software License, Version 2.0) jackson-module-scala (com.fasterxml.jackson.module:jackson-module-scala_2.13:2.16.1 - https://github.com/FasterXML/jackson-module-scala)
+     (Apache License, Version 2.0) Caffeine cache (com.github.ben-manes.caffeine:caffeine:3.1.5 - https://github.com/ben-manes/caffeine)
+     (BSD 2-Clause License) zstd-jni (com.github.luben:zstd-jni:1.5.5-6 - https://github.com/luben/zstd-jni)
      (The Apache Software License, Version 2.0) bucket4j-core (com.github.vladimir-bukhtoyarov:bucket4j-core:7.6.0 - http://github.com/vladimir-bukhtoyarov/bucket4j/bucket4j-core)
      (The Apache Software License, Version 2.0) FindBugs-jsr305 (com.google.code.findbugs:jsr305:3.0.2 - http://findbugs.sourceforge.net/)
      (Apache-2.0) Gson (com.google.code.gson:gson:2.10.1 - https://github.com/google/gson/gson)
@@ -27,11 +28,16 @@ Lists of 234 third-party dependencies.
      (Apache 2.0) OkHttp (com.squareup.okhttp3:okhttp:3.14.9 - https://github.com/square/okhttp/okhttp)
      (Apache 2.0) Okio (com.squareup.okio:okio:1.17.2 - https://github.com/square/okio/okio)
      (BSD) ParaNamer Core (com.thoughtworks.paranamer:paranamer:2.8 - https://github.com/paul-hammant/paranamer/paranamer)
-     (Apache 2.0 License) scala-logging (com.typesafe.scala-logging:scala-logging_2.12:3.9.4 - https://github.com/lightbend/scala-logging)
+     (Apache 2.0 License) scala-logging (com.typesafe.scala-logging:scala-logging_2.13:3.9.4 - https://github.com/lightbend/scala-logging)
      (Apache License 2.0) Metrics Core Library (com.yammer.metrics:metrics-core:2.2.0 - http://metrics.codahale.com/metrics-core/)
+     (Apache License, Version 2.0) Apache Commons BeanUtils (commons-beanutils:commons-beanutils:1.9.4 - https://commons.apache.org/proper/commons-beanutils/)
      (Apache License, Version 2.0) Apache Commons CLI (commons-cli:commons-cli:1.4 - http://commons.apache.org/proper/commons-cli/)
-     (Apache-2.0) Apache Commons Codec (commons-codec:commons-codec:1.16.1 - https://commons.apache.org/proper/commons-codec/)
      (Apache-2.0) Apache Commons Codec (commons-codec:commons-codec:1.17.0 - https://commons.apache.org/proper/commons-codec/)
+     (Apache License, Version 2.0) Apache Commons Collections (commons-collections:commons-collections:3.2.2 - http://commons.apache.org/collections/)
+     (The Apache Software License, Version 2.0) Commons Digester (commons-digester:commons-digester:2.1 - http://commons.apache.org/digester/)
+     (Apache-2.0) Apache Commons IO (commons-io:commons-io:2.15.1 - https://commons.apache.org/proper/commons-io/)
+     (The Apache Software License, Version 2.0) Apache Commons Logging (commons-logging:commons-logging:1.2 - http://commons.apache.org/proper/commons-logging/)
+     (Apache License, Version 2.0) Apache Commons Validator (commons-validator:commons-validator:1.7 - http://commons.apache.org/proper/commons-validator/)
      (Unknown license) contract (dev.knative.eventing.kafka.broker:contract:1.0-SNAPSHOT - no url defined)
      (Unknown license) core (dev.knative.eventing.kafka.broker:core:1.0-SNAPSHOT - no url defined)
      (Unknown license) dispatcher (dev.knative.eventing.kafka.broker:dispatcher:1.0-SNAPSHOT - no url defined)
@@ -44,8 +50,8 @@ Lists of 234 third-party dependencies.
      (The Apache Software License, Version 2.0) CloudEvents - JSON Jackson (io.cloudevents:cloudevents-json-jackson:2.5.0 - https://cloudevents.github.io/sdk-java/cloudevents-json-jackson/)
      (The Apache Software License, Version 2.0) CloudEvents - Kafka Binding (io.cloudevents:cloudevents-kafka:2.5.0 - https://cloudevents.github.io/sdk-java/cloudevents-kafka/)
      (The Apache Software License, Version 2.0) cloudevents-sql (io.cloudevents:cloudevents-sql:2.5.0 - https://cloudevents.github.io/sdk-java/cloudevents-sql/)
-     (Apache Software License 2.0) Debezium API (io.debezium:debezium-api:1.9.6.Final - https://debezium.io/debezium-parent/debezium-api)
-     (Apache Software License 2.0) Debezium Core (io.debezium:debezium-core:1.9.6.Final - https://debezium.io/debezium-parent/debezium-core)
+     (Apache Software License 2.0) Debezium API (io.debezium:debezium-api:2.6.1.Final - https://debezium.io/debezium-parent/debezium-api)
+     (Apache Software License 2.0) Debezium Core (io.debezium:debezium-core:2.6.1.Final - https://debezium.io/debezium-parent/debezium-core)
      (Apache License 2.0) Metrics Core (io.dropwizard.metrics:metrics-core:4.1.12.1 - https://metrics.dropwizard.io/metrics-core)
      (Apache License, Version 2.0) Fabric8 :: Kubernetes :: Java Client (io.fabric8:kubernetes-client:6.10.0 - https://github.com/fabric8io/kubernetes-client/kubernetes-client)
      (Apache License, Version 2.0) Fabric8 :: Kubernetes :: Java Client API (io.fabric8:kubernetes-client-api:6.10.0 - https://github.com/fabric8io/kubernetes-client/kubernetes-client-api)
@@ -187,16 +193,19 @@ Lists of 234 third-party dependencies.
      (MIT) argparse4j (net.sourceforge.argparse4j:argparse4j:0.7.0 - http://argparse4j.github.io)
      (The BSD License) ANTLR 4 Runtime (org.antlr:antlr4-runtime:4.9.2 - http://www.antlr.org/antlr4-runtime)
      (Apache License, Version 2.0) Apache Commons Math (org.apache.commons:commons-math3:3.6.1 - http://commons.apache.org/proper/commons-math/)
-     (The Apache License, Version 2.0) Apache Kafka (org.apache.kafka:kafka-clients:3.2.3 - https://kafka.apache.org)
-     (The Apache License, Version 2.0) Apache Kafka (org.apache.kafka:kafka-metadata:3.2.3 - https://kafka.apache.org)
-     (The Apache License, Version 2.0) Apache Kafka (org.apache.kafka:kafka-raft:3.2.3 - https://kafka.apache.org)
-     (The Apache License, Version 2.0) Apache Kafka (org.apache.kafka:kafka-server-common:3.2.3 - https://kafka.apache.org)
-     (The Apache License, Version 2.0) Apache Kafka (org.apache.kafka:kafka-storage:3.2.3 - https://kafka.apache.org)
-     (The Apache License, Version 2.0) Apache Kafka (org.apache.kafka:kafka-storage-api:3.2.3 - https://kafka.apache.org)
-     (The Apache License, Version 2.0) Apache Kafka (org.apache.kafka:kafka_2.12:3.2.3 - https://kafka.apache.org)
-     (Apache License, Version 2.0) Apache Yetus - Audience Annotations (org.apache.yetus:audience-annotations:0.5.0 - https://yetus.apache.org/audience-annotations)
-     (Apache License, Version 2.0) Apache ZooKeeper - Server (org.apache.zookeeper:zookeeper:3.6.3 - http://zookeeper.apache.org/zookeeper)
-     (Apache License, Version 2.0) Apache ZooKeeper - Jute (org.apache.zookeeper:zookeeper-jute:3.6.3 - http://zookeeper.apache.org/zookeeper-jute)
+     (The Apache License, Version 2.0) Apache Kafka (org.apache.kafka:kafka-clients:3.7.0 - https://kafka.apache.org)
+     (The Apache License, Version 2.0) Apache Kafka (org.apache.kafka:kafka-group-coordinator:3.7.0 - https://kafka.apache.org)
+     (The Apache License, Version 2.0) Apache Kafka (org.apache.kafka:kafka-metadata:3.7.0 - https://kafka.apache.org)
+     (The Apache License, Version 2.0) Apache Kafka (org.apache.kafka:kafka-raft:3.7.0 - https://kafka.apache.org)
+     (The Apache License, Version 2.0) Apache Kafka (org.apache.kafka:kafka-server:3.7.0 - https://kafka.apache.org)
+     (The Apache License, Version 2.0) Apache Kafka (org.apache.kafka:kafka-server-common:3.7.0 - https://kafka.apache.org)
+     (The Apache License, Version 2.0) Apache Kafka (org.apache.kafka:kafka-storage:3.7.0 - https://kafka.apache.org)
+     (The Apache License, Version 2.0) Apache Kafka (org.apache.kafka:kafka-storage-api:3.7.0 - https://kafka.apache.org)
+     (The Apache License, Version 2.0) Apache Kafka (org.apache.kafka:kafka-tools-api:3.7.0 - https://kafka.apache.org)
+     (The Apache License, Version 2.0) Apache Kafka (org.apache.kafka:kafka_2.13:3.7.0 - https://kafka.apache.org)
+     (Apache License, Version 2.0) Apache Yetus - Audience Annotations (org.apache.yetus:audience-annotations:0.12.0 - https://yetus.apache.org/audience-annotations)
+     (Apache License, Version 2.0) Apache ZooKeeper - Server (org.apache.zookeeper:zookeeper:3.8.3 - http://zookeeper.apache.org/zookeeper)
+     (Apache License, Version 2.0) Apache ZooKeeper - Jute (org.apache.zookeeper:zookeeper-jute:3.8.3 - http://zookeeper.apache.org/zookeeper-jute)
      (The Apache License, Version 2.0) org.apiguardian:apiguardian-api (org.apiguardian:apiguardian-api:1.1.2 - https://github.com/apiguardian-team/apiguardian)
      (Apache License, Version 2.0) AssertJ Core (org.assertj:assertj-core:3.26.0 - https://assertj.github.io/doc/#assertj-core)
      (Apache 2.0) Awaitility (org.awaitility:awaitility:4.2.0 - http://awaitility.org)
@@ -226,9 +235,10 @@ Lists of 234 third-party dependencies.
      (GNU General Public License (GPL), version 2, with the Classpath exception) JMH Core (org.openjdk.jmh:jmh-core:1.37 - http://openjdk.java.net/projects/code-tools/jmh/jmh-core/)
      (GNU General Public License (GPL), version 2, with the Classpath exception) JMH Generators: Annotation Processors (org.openjdk.jmh:jmh-generator-annprocess:1.37 - http://openjdk.java.net/projects/code-tools/jmh/jmh-generator-annprocess/)
      (The Apache License, Version 2.0) org.opentest4j:opentest4j (org.opentest4j:opentest4j:1.3.0 - https://github.com/ota4j-team/opentest4j)
-     (Apache-2.0) Scala Library (org.scala-lang:scala-library:2.12.15 - https://www.scala-lang.org/)
-     (Apache-2.0) scala-collection-compat (org.scala-lang.modules:scala-collection-compat_2.12:2.6.0 - http://www.scala-lang.org/)
-     (Apache-2.0) scala-java8-compat (org.scala-lang.modules:scala-java8-compat_2.12:1.0.2 - http://www.scala-lang.org/)
+     (The MIT License) PCollections (org.pcollections:pcollections:4.0.1 - https://github.com/hrldcpr/pcollections)
+     (Apache-2.0) Scala Library (org.scala-lang:scala-library:2.13.14 - https://www.scala-lang.org/)
+     (Apache-2.0) scala-collection-compat (org.scala-lang.modules:scala-collection-compat_2.13:2.10.0 - http://www.scala-lang.org/)
+     (Apache-2.0) scala-java8-compat (org.scala-lang.modules:scala-java8-compat_2.13:1.0.2 - http://www.scala-lang.org/)
      (MIT License) SLF4J API Module (org.slf4j:slf4j-api:2.0.9 - http://www.slf4j.org)
      (Apache License, Version 2.0) SnakeYAML Engine (org.snakeyaml:snakeyaml-engine:2.7 - https://bitbucket.org/snakeyaml/snakeyaml-engine)
      (Apache License 2.0) wildfly-common (org.wildfly.common:wildfly-common:1.7.0.Final - http://www.jboss.org)

diff --git a/...src/main/java/dev/knative/eventing/kafka/broker/dispatcher/impl/RecordDispatcherImpl.java b/...src/main/java/dev/knative/eventing/kafka/broker/dispatcher/impl/RecordDispatcherImpl.java
@@ -362,7 +362,7 @@ private ConsumerRecordContext maybeDeserializeValueFromHeaders(ConsumerRecordCon
         // headers even though the record is a valid CloudEvent.
         logDebug("Value is null", recordContext.getRecord());
         final var value = cloudEventDeserializer.deserialize(
-                recordContext.getRecord().topic(), recordContext.getRecord().headers(), null);
+                recordContext.getRecord().topic(), recordContext.getRecord().headers(), (byte[]) null);
         recordContext.setRecord(KafkaConsumerRecordUtils.copyRecordAssigningValue(recordContext.getRecord(), value));
         return recordContext;
     }

diff --git a/...dev/knative/eventing/kafka/broker/dispatcher/impl/consumer/NullCloudEventInterceptor.java b/...dev/knative/eventing/kafka/broker/dispatcher/impl/consumer/NullCloudEventInterceptor.java
@@ -82,6 +82,6 @@ private ConsumerRecord<Object, CloudEvent> maybeDeserializeRecord(ConsumerRecord
         // headers even though the record is a valid CloudEvent.
         logger.debug("deserializing null record");
         return KafkaConsumerRecordUtils.copyRecordAssigningValue(
-                record, cloudEventDeserializer.deserialize(record.topic(), record.headers(), null));
+                record, cloudEventDeserializer.deserialize(record.topic(), record.headers(), (byte[]) null));
     }
 }
diff --git a/...st/java/dev/knative/eventing/kafka/broker/dispatcher/impl/consumer/OffsetManagerTest.java b/...st/java/dev/knative/eventing/kafka/broker/dispatcher/impl/consumer/OffsetManagerTest.java
@@ -334,7 +334,7 @@ public void testFailedCommitRetry(final Vertx vertx) {
         final ReactiveKafkaConsumer<String, CloudEvent> consumer = mock(ReactiveKafkaConsumer.class);
         when(consumer.commit((Map<TopicPartition, OffsetAndMetadata>) any())).then(invocationOnMock -> {
             if (counter.incrementAndGet() == 1) {
-                return Future.failedFuture(new RuntimeException());
+                return Future.failedFuture(new RuntimeException("throw exception for testing"));
             }
             return Future.succeededFuture();
         });
@@ -359,7 +359,7 @@ public void testAlwaysFailedCommit(final Vertx vertx) {
 
         final ReactiveKafkaConsumer<String, CloudEvent> consumer = mock(ReactiveKafkaConsumer.class);
         when(consumer.commit((Map<TopicPartition, OffsetAndMetadata>) any()))
-                .then(invocationOnMock -> Future.failedFuture(new RuntimeException()));
+                .then(invocationOnMock -> Future.failedFuture(new RuntimeException("throw exception for testing")));
 
         final var r = record("aaa", 0, 0);
 

diff --git a/data-plane/pom.xml b/data-plane/pom.xml
@@ -36,7 +36,7 @@
     <maven.surefire.plugin.version>3.2.5</maven.surefire.plugin.version>
     <maven.shade.plugin.version>3.5.1</maven.shade.plugin.version>
     <maven.editorconfig.plugin.version>0.1.1</maven.editorconfig.plugin.version>
-    <maven.jacoco.plugin.version>0.8.10</maven.jacoco.plugin.version>
+    <maven.jacoco.plugin.version>0.8.12</maven.jacoco.plugin.version>
     <maven.enforcer.plugin.version>3.4.1</maven.enforcer.plugin.version>
     <maven.license.plugin.version>2.0.0</maven.license.plugin.version>
     <mycila.license.plugin.version>4.1</mycila.license.plugin.version>
@@ -58,8 +58,8 @@
     <junit.jupiter.version>5.10.1</junit.jupiter.version>
     <mokito.junit.jupiter.version>5.12.0</mokito.junit.jupiter.version>
     <fabric8.kubernetes.version>6.10.0</fabric8.kubernetes.version>
-    <kafka.version>3.2.3</kafka.version>
-    <debezium.version>1.9.6.Final</debezium.version>
+    <kafka.version>3.7.0</kafka.version>
+    <debezium.version>2.6.1.Final</debezium.version>
     <jib.version>3.4.0</jib.version>
     <quarkus.version>3.8.4</quarkus.version>
     <antlr.version>4.9.2
@@ -398,7 +398,7 @@
       </dependency>
       <dependency>
         <groupId>org.apache.kafka</groupId>
-        <artifactId>kafka_2.12</artifactId>
+        <artifactId>kafka_2.13</artifactId>
         <version>${kafka.version}</version>
         <scope>test</scope>
         <exclusions>
@@ -415,13 +415,13 @@
       <dependency>
         <groupId>org.scala-lang</groupId>
         <artifactId>scala-library</artifactId>
-        <version>2.12.15</version>
+        <version>2.13.14</version>
         <scope>test</scope>
       </dependency>
       <dependency>
         <groupId>org.scala-lang</groupId>
         <artifactId>scala-reflect</artifactId>
-        <version>2.13.12</version>
+        <version>2.13.14</version>
         <scope>test</scope>
       </dependency>
     </dependencies>

diff --git a/data-plane/tests/pom.xml b/data-plane/tests/pom.xml
@@ -95,7 +95,7 @@
     </dependency>
     <dependency>
       <groupId>org.apache.kafka</groupId>
-      <artifactId>kafka_2.12</artifactId>
+      <artifactId>kafka_2.13</artifactId>
       <scope>test</scope>
     </dependency>
     <dependency>