From 7d14a4ae59a87ecebf647e453f6e4a415e91cc4a Mon Sep 17 00:00:00 2001 From: Pierangelo Di Pilato Date: Thu, 6 Jun 2024 13:26:56 +0200 Subject: [PATCH] Bump to latest Kafka clients 3.7 (#3900) * Bump to latest Kafka clients Signed-off-by: Pierangelo Di Pilato * Fix compilation errors Signed-off-by: Pierangelo Di Pilato --------- Signed-off-by: Pierangelo Di Pilato --- data-plane/THIRD-PARTY.txt | 50 +++++++++++-------- .../dispatcher/impl/RecordDispatcherImpl.java | 2 +- .../consumer/NullCloudEventInterceptor.java | 2 +- .../impl/consumer/OffsetManagerTest.java | 4 +- data-plane/pom.xml | 12 ++--- data-plane/tests/pom.xml | 2 +- 6 files changed, 41 insertions(+), 31 deletions(-) diff --git a/data-plane/THIRD-PARTY.txt b/data-plane/THIRD-PARTY.txt index b809de54c4..e943ed27ee 100644 --- a/data-plane/THIRD-PARTY.txt +++ b/data-plane/THIRD-PARTY.txt @@ -1,5 +1,5 @@ -Lists of 234 third-party dependencies. +Lists of 245 third-party dependencies. (Eclipse Public License - v 1.0) (GNU Lesser General Public License) Logback Classic Module (ch.qos.logback:logback-classic:1.4.14 - http://logback.qos.ch/logback-classic) (Eclipse Public License - v 1.0) (GNU Lesser General Public License) Logback Core Module (ch.qos.logback:logback-core:1.4.14 - http://logback.qos.ch/logback-core) (Apache License 2.0) brotli4j (com.aayushatharva.brotli4j:brotli4j:1.16.0 - https://github.com/hyperxpro/Brotli4j/brotli4j) @@ -12,8 +12,9 @@ Lists of 234 third-party dependencies. (The Apache Software License, Version 2.0) Jackson-dataformat-YAML (com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.16.1 - https://github.com/FasterXML/jackson-dataformats-text) (The Apache Software License, Version 2.0) Jackson datatype: jdk8 (com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.16.1 - https://github.com/FasterXML/jackson-modules-java8/jackson-datatype-jdk8) (The Apache Software License, Version 2.0) Jackson datatype: JSR310 (com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.16.1 - https://github.com/FasterXML/jackson-modules-java8/jackson-datatype-jsr310) - (The Apache Software License, Version 2.0) jackson-module-scala (com.fasterxml.jackson.module:jackson-module-scala_2.12:2.16.1 - https://github.com/FasterXML/jackson-module-scala) - (BSD 2-Clause License) zstd-jni (com.github.luben:zstd-jni:1.5.2-1 - https://github.com/luben/zstd-jni) + (The Apache Software License, Version 2.0) jackson-module-scala (com.fasterxml.jackson.module:jackson-module-scala_2.13:2.16.1 - https://github.com/FasterXML/jackson-module-scala) + (Apache License, Version 2.0) Caffeine cache (com.github.ben-manes.caffeine:caffeine:3.1.5 - https://github.com/ben-manes/caffeine) + (BSD 2-Clause License) zstd-jni (com.github.luben:zstd-jni:1.5.5-6 - https://github.com/luben/zstd-jni) (The Apache Software License, Version 2.0) bucket4j-core (com.github.vladimir-bukhtoyarov:bucket4j-core:7.6.0 - http://github.com/vladimir-bukhtoyarov/bucket4j/bucket4j-core) (The Apache Software License, Version 2.0) FindBugs-jsr305 (com.google.code.findbugs:jsr305:3.0.2 - http://findbugs.sourceforge.net/) (Apache-2.0) Gson (com.google.code.gson:gson:2.10.1 - https://github.com/google/gson/gson) @@ -27,11 +28,16 @@ Lists of 234 third-party dependencies. (Apache 2.0) OkHttp (com.squareup.okhttp3:okhttp:3.14.9 - https://github.com/square/okhttp/okhttp) (Apache 2.0) Okio (com.squareup.okio:okio:1.17.2 - https://github.com/square/okio/okio) (BSD) ParaNamer Core (com.thoughtworks.paranamer:paranamer:2.8 - https://github.com/paul-hammant/paranamer/paranamer) - (Apache 2.0 License) scala-logging (com.typesafe.scala-logging:scala-logging_2.12:3.9.4 - https://github.com/lightbend/scala-logging) + (Apache 2.0 License) scala-logging (com.typesafe.scala-logging:scala-logging_2.13:3.9.4 - https://github.com/lightbend/scala-logging) (Apache License 2.0) Metrics Core Library (com.yammer.metrics:metrics-core:2.2.0 - http://metrics.codahale.com/metrics-core/) + (Apache License, Version 2.0) Apache Commons BeanUtils (commons-beanutils:commons-beanutils:1.9.4 - https://commons.apache.org/proper/commons-beanutils/) (Apache License, Version 2.0) Apache Commons CLI (commons-cli:commons-cli:1.4 - http://commons.apache.org/proper/commons-cli/) - (Apache-2.0) Apache Commons Codec (commons-codec:commons-codec:1.16.1 - https://commons.apache.org/proper/commons-codec/) (Apache-2.0) Apache Commons Codec (commons-codec:commons-codec:1.17.0 - https://commons.apache.org/proper/commons-codec/) + (Apache License, Version 2.0) Apache Commons Collections (commons-collections:commons-collections:3.2.2 - http://commons.apache.org/collections/) + (The Apache Software License, Version 2.0) Commons Digester (commons-digester:commons-digester:2.1 - http://commons.apache.org/digester/) + (Apache-2.0) Apache Commons IO (commons-io:commons-io:2.15.1 - https://commons.apache.org/proper/commons-io/) + (The Apache Software License, Version 2.0) Apache Commons Logging (commons-logging:commons-logging:1.2 - http://commons.apache.org/proper/commons-logging/) + (Apache License, Version 2.0) Apache Commons Validator (commons-validator:commons-validator:1.7 - http://commons.apache.org/proper/commons-validator/) (Unknown license) contract (dev.knative.eventing.kafka.broker:contract:1.0-SNAPSHOT - no url defined) (Unknown license) core (dev.knative.eventing.kafka.broker:core:1.0-SNAPSHOT - no url defined) (Unknown license) dispatcher (dev.knative.eventing.kafka.broker:dispatcher:1.0-SNAPSHOT - no url defined) @@ -44,8 +50,8 @@ Lists of 234 third-party dependencies. (The Apache Software License, Version 2.0) CloudEvents - JSON Jackson (io.cloudevents:cloudevents-json-jackson:2.5.0 - https://cloudevents.github.io/sdk-java/cloudevents-json-jackson/) (The Apache Software License, Version 2.0) CloudEvents - Kafka Binding (io.cloudevents:cloudevents-kafka:2.5.0 - https://cloudevents.github.io/sdk-java/cloudevents-kafka/) (The Apache Software License, Version 2.0) cloudevents-sql (io.cloudevents:cloudevents-sql:2.5.0 - https://cloudevents.github.io/sdk-java/cloudevents-sql/) - (Apache Software License 2.0) Debezium API (io.debezium:debezium-api:1.9.6.Final - https://debezium.io/debezium-parent/debezium-api) - (Apache Software License 2.0) Debezium Core (io.debezium:debezium-core:1.9.6.Final - https://debezium.io/debezium-parent/debezium-core) + (Apache Software License 2.0) Debezium API (io.debezium:debezium-api:2.6.1.Final - https://debezium.io/debezium-parent/debezium-api) + (Apache Software License 2.0) Debezium Core (io.debezium:debezium-core:2.6.1.Final - https://debezium.io/debezium-parent/debezium-core) (Apache License 2.0) Metrics Core (io.dropwizard.metrics:metrics-core:4.1.12.1 - https://metrics.dropwizard.io/metrics-core) (Apache License, Version 2.0) Fabric8 :: Kubernetes :: Java Client (io.fabric8:kubernetes-client:6.10.0 - https://github.com/fabric8io/kubernetes-client/kubernetes-client) (Apache License, Version 2.0) Fabric8 :: Kubernetes :: Java Client API (io.fabric8:kubernetes-client-api:6.10.0 - https://github.com/fabric8io/kubernetes-client/kubernetes-client-api) @@ -187,16 +193,19 @@ Lists of 234 third-party dependencies. (MIT) argparse4j (net.sourceforge.argparse4j:argparse4j:0.7.0 - http://argparse4j.github.io) (The BSD License) ANTLR 4 Runtime (org.antlr:antlr4-runtime:4.9.2 - http://www.antlr.org/antlr4-runtime) (Apache License, Version 2.0) Apache Commons Math (org.apache.commons:commons-math3:3.6.1 - http://commons.apache.org/proper/commons-math/) - (The Apache License, Version 2.0) Apache Kafka (org.apache.kafka:kafka-clients:3.2.3 - https://kafka.apache.org) - (The Apache License, Version 2.0) Apache Kafka (org.apache.kafka:kafka-metadata:3.2.3 - https://kafka.apache.org) - (The Apache License, Version 2.0) Apache Kafka (org.apache.kafka:kafka-raft:3.2.3 - https://kafka.apache.org) - (The Apache License, Version 2.0) Apache Kafka (org.apache.kafka:kafka-server-common:3.2.3 - https://kafka.apache.org) - (The Apache License, Version 2.0) Apache Kafka (org.apache.kafka:kafka-storage:3.2.3 - https://kafka.apache.org) - (The Apache License, Version 2.0) Apache Kafka (org.apache.kafka:kafka-storage-api:3.2.3 - https://kafka.apache.org) - (The Apache License, Version 2.0) Apache Kafka (org.apache.kafka:kafka_2.12:3.2.3 - https://kafka.apache.org) - (Apache License, Version 2.0) Apache Yetus - Audience Annotations (org.apache.yetus:audience-annotations:0.5.0 - https://yetus.apache.org/audience-annotations) - (Apache License, Version 2.0) Apache ZooKeeper - Server (org.apache.zookeeper:zookeeper:3.6.3 - http://zookeeper.apache.org/zookeeper) - (Apache License, Version 2.0) Apache ZooKeeper - Jute (org.apache.zookeeper:zookeeper-jute:3.6.3 - http://zookeeper.apache.org/zookeeper-jute) + (The Apache License, Version 2.0) Apache Kafka (org.apache.kafka:kafka-clients:3.7.0 - https://kafka.apache.org) + (The Apache License, Version 2.0) Apache Kafka (org.apache.kafka:kafka-group-coordinator:3.7.0 - https://kafka.apache.org) + (The Apache License, Version 2.0) Apache Kafka (org.apache.kafka:kafka-metadata:3.7.0 - https://kafka.apache.org) + (The Apache License, Version 2.0) Apache Kafka (org.apache.kafka:kafka-raft:3.7.0 - https://kafka.apache.org) + (The Apache License, Version 2.0) Apache Kafka (org.apache.kafka:kafka-server:3.7.0 - https://kafka.apache.org) + (The Apache License, Version 2.0) Apache Kafka (org.apache.kafka:kafka-server-common:3.7.0 - https://kafka.apache.org) + (The Apache License, Version 2.0) Apache Kafka (org.apache.kafka:kafka-storage:3.7.0 - https://kafka.apache.org) + (The Apache License, Version 2.0) Apache Kafka (org.apache.kafka:kafka-storage-api:3.7.0 - https://kafka.apache.org) + (The Apache License, Version 2.0) Apache Kafka (org.apache.kafka:kafka-tools-api:3.7.0 - https://kafka.apache.org) + (The Apache License, Version 2.0) Apache Kafka (org.apache.kafka:kafka_2.13:3.7.0 - https://kafka.apache.org) + (Apache License, Version 2.0) Apache Yetus - Audience Annotations (org.apache.yetus:audience-annotations:0.12.0 - https://yetus.apache.org/audience-annotations) + (Apache License, Version 2.0) Apache ZooKeeper - Server (org.apache.zookeeper:zookeeper:3.8.3 - http://zookeeper.apache.org/zookeeper) + (Apache License, Version 2.0) Apache ZooKeeper - Jute (org.apache.zookeeper:zookeeper-jute:3.8.3 - http://zookeeper.apache.org/zookeeper-jute) (The Apache License, Version 2.0) org.apiguardian:apiguardian-api (org.apiguardian:apiguardian-api:1.1.2 - https://github.com/apiguardian-team/apiguardian) (Apache License, Version 2.0) AssertJ Core (org.assertj:assertj-core:3.26.0 - https://assertj.github.io/doc/#assertj-core) (Apache 2.0) Awaitility (org.awaitility:awaitility:4.2.0 - http://awaitility.org) @@ -226,9 +235,10 @@ Lists of 234 third-party dependencies. (GNU General Public License (GPL), version 2, with the Classpath exception) JMH Core (org.openjdk.jmh:jmh-core:1.37 - http://openjdk.java.net/projects/code-tools/jmh/jmh-core/) (GNU General Public License (GPL), version 2, with the Classpath exception) JMH Generators: Annotation Processors (org.openjdk.jmh:jmh-generator-annprocess:1.37 - http://openjdk.java.net/projects/code-tools/jmh/jmh-generator-annprocess/) (The Apache License, Version 2.0) org.opentest4j:opentest4j (org.opentest4j:opentest4j:1.3.0 - https://github.com/ota4j-team/opentest4j) - (Apache-2.0) Scala Library (org.scala-lang:scala-library:2.12.15 - https://www.scala-lang.org/) - (Apache-2.0) scala-collection-compat (org.scala-lang.modules:scala-collection-compat_2.12:2.6.0 - http://www.scala-lang.org/) - (Apache-2.0) scala-java8-compat (org.scala-lang.modules:scala-java8-compat_2.12:1.0.2 - http://www.scala-lang.org/) + (The MIT License) PCollections (org.pcollections:pcollections:4.0.1 - https://github.com/hrldcpr/pcollections) + (Apache-2.0) Scala Library (org.scala-lang:scala-library:2.13.14 - https://www.scala-lang.org/) + (Apache-2.0) scala-collection-compat (org.scala-lang.modules:scala-collection-compat_2.13:2.10.0 - http://www.scala-lang.org/) + (Apache-2.0) scala-java8-compat (org.scala-lang.modules:scala-java8-compat_2.13:1.0.2 - http://www.scala-lang.org/) (MIT License) SLF4J API Module (org.slf4j:slf4j-api:2.0.9 - http://www.slf4j.org) (Apache License, Version 2.0) SnakeYAML Engine (org.snakeyaml:snakeyaml-engine:2.7 - https://bitbucket.org/snakeyaml/snakeyaml-engine) (Apache License 2.0) wildfly-common (org.wildfly.common:wildfly-common:1.7.0.Final - http://www.jboss.org) diff --git a/data-plane/dispatcher/src/main/java/dev/knative/eventing/kafka/broker/dispatcher/impl/RecordDispatcherImpl.java b/data-plane/dispatcher/src/main/java/dev/knative/eventing/kafka/broker/dispatcher/impl/RecordDispatcherImpl.java index d36620d686..d7da7083b8 100644 --- a/data-plane/dispatcher/src/main/java/dev/knative/eventing/kafka/broker/dispatcher/impl/RecordDispatcherImpl.java +++ b/data-plane/dispatcher/src/main/java/dev/knative/eventing/kafka/broker/dispatcher/impl/RecordDispatcherImpl.java @@ -362,7 +362,7 @@ private ConsumerRecordContext maybeDeserializeValueFromHeaders(ConsumerRecordCon // headers even though the record is a valid CloudEvent. logDebug("Value is null", recordContext.getRecord()); final var value = cloudEventDeserializer.deserialize( - recordContext.getRecord().topic(), recordContext.getRecord().headers(), null); + recordContext.getRecord().topic(), recordContext.getRecord().headers(), (byte[]) null); recordContext.setRecord(KafkaConsumerRecordUtils.copyRecordAssigningValue(recordContext.getRecord(), value)); return recordContext; } diff --git a/data-plane/dispatcher/src/main/java/dev/knative/eventing/kafka/broker/dispatcher/impl/consumer/NullCloudEventInterceptor.java b/data-plane/dispatcher/src/main/java/dev/knative/eventing/kafka/broker/dispatcher/impl/consumer/NullCloudEventInterceptor.java index 12ab37e476..cc1326f43a 100644 --- a/data-plane/dispatcher/src/main/java/dev/knative/eventing/kafka/broker/dispatcher/impl/consumer/NullCloudEventInterceptor.java +++ b/data-plane/dispatcher/src/main/java/dev/knative/eventing/kafka/broker/dispatcher/impl/consumer/NullCloudEventInterceptor.java @@ -82,6 +82,6 @@ private ConsumerRecord maybeDeserializeRecord(ConsumerRecord // headers even though the record is a valid CloudEvent. logger.debug("deserializing null record"); return KafkaConsumerRecordUtils.copyRecordAssigningValue( - record, cloudEventDeserializer.deserialize(record.topic(), record.headers(), null)); + record, cloudEventDeserializer.deserialize(record.topic(), record.headers(), (byte[]) null)); } } diff --git a/data-plane/dispatcher/src/test/java/dev/knative/eventing/kafka/broker/dispatcher/impl/consumer/OffsetManagerTest.java b/data-plane/dispatcher/src/test/java/dev/knative/eventing/kafka/broker/dispatcher/impl/consumer/OffsetManagerTest.java index d77cca0f43..3da97130c6 100644 --- a/data-plane/dispatcher/src/test/java/dev/knative/eventing/kafka/broker/dispatcher/impl/consumer/OffsetManagerTest.java +++ b/data-plane/dispatcher/src/test/java/dev/knative/eventing/kafka/broker/dispatcher/impl/consumer/OffsetManagerTest.java @@ -334,7 +334,7 @@ public void testFailedCommitRetry(final Vertx vertx) { final ReactiveKafkaConsumer consumer = mock(ReactiveKafkaConsumer.class); when(consumer.commit((Map) any())).then(invocationOnMock -> { if (counter.incrementAndGet() == 1) { - return Future.failedFuture(new RuntimeException()); + return Future.failedFuture(new RuntimeException("throw exception for testing")); } return Future.succeededFuture(); }); @@ -359,7 +359,7 @@ public void testAlwaysFailedCommit(final Vertx vertx) { final ReactiveKafkaConsumer consumer = mock(ReactiveKafkaConsumer.class); when(consumer.commit((Map) any())) - .then(invocationOnMock -> Future.failedFuture(new RuntimeException())); + .then(invocationOnMock -> Future.failedFuture(new RuntimeException("throw exception for testing"))); final var r = record("aaa", 0, 0); diff --git a/data-plane/pom.xml b/data-plane/pom.xml index 7ddb9614dc..85dbe5531a 100644 --- a/data-plane/pom.xml +++ b/data-plane/pom.xml @@ -36,7 +36,7 @@ 3.2.5 3.5.1 0.1.1 - 0.8.10 + 0.8.12 3.4.1 2.0.0 4.1 @@ -58,8 +58,8 @@ 5.10.1 5.12.0 6.10.0 - 3.2.3 - 1.9.6.Final + 3.7.0 + 2.6.1.Final 3.4.0 3.8.4 4.9.2 @@ -398,7 +398,7 @@ org.apache.kafka - kafka_2.12 + kafka_2.13 ${kafka.version} test @@ -415,13 +415,13 @@ org.scala-lang scala-library - 2.12.15 + 2.13.14 test org.scala-lang scala-reflect - 2.13.12 + 2.13.14 test diff --git a/data-plane/tests/pom.xml b/data-plane/tests/pom.xml index d692cb841d..e7abe388ca 100644 --- a/data-plane/tests/pom.xml +++ b/data-plane/tests/pom.xml @@ -95,7 +95,7 @@ org.apache.kafka - kafka_2.12 + kafka_2.13 test