From 60bc158a9df69979ac68be950a3c60c697337d32 Mon Sep 17 00:00:00 2001 From: deko2369 Date: Sat, 15 Jun 2024 09:48:24 +0000 Subject: [PATCH 1/2] format test file --- .../embulk/output/s3_parquet/EmbulkPluginTestHelper.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/test/scala/org/embulk/output/s3_parquet/EmbulkPluginTestHelper.scala b/src/test/scala/org/embulk/output/s3_parquet/EmbulkPluginTestHelper.scala index c532b34..cfd8666 100644 --- a/src/test/scala/org/embulk/output/s3_parquet/EmbulkPluginTestHelper.scala +++ b/src/test/scala/org/embulk/output/s3_parquet/EmbulkPluginTestHelper.scala @@ -237,9 +237,11 @@ abstract class EmbulkPluginTestHelper ) .build() - Iterator.continually(reader.read()).takeWhile(_ != null).map( - record => record.getSchema.getFields.map(f => record.get(f.name())) - ).toSeq + Iterator + .continually(reader.read()) + .takeWhile(_ != null) + .map(record => record.getSchema.getFields.map(f => record.get(f.name()))) + .toSeq } def loadConfigSourceFromYamlString(yaml: String): ConfigSource = From b86981506fc0906367a4d0e697d4fa6ea4d6a00d Mon Sep 17 00:00:00 2001 From: deko2369 Date: Sat, 15 Jun 2024 09:49:34 +0000 Subject: [PATCH 2/2] update hadoop-common version --- build.gradle | 6 +-- .../embulkPluginRuntime.lockfile | 40 ++++++++++--------- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/build.gradle b/build.gradle index f0a56e9..fd636b5 100644 --- a/build.gradle +++ b/build.gradle @@ -11,7 +11,7 @@ repositories { } group = "pro.civitaspo" -version = "0.5.2" +version = "0.5.3" description = "Dumps records to S3 Parquet." sourceCompatibility = 1.8 @@ -39,7 +39,7 @@ dependencies { compile('org.apache.parquet:parquet-format:2.7.0') { exclude group: 'org.slf4j', module: 'slf4j-api' } - compile('org.apache.hadoop:hadoop-common:2.9.2') { + compile('org.apache.hadoop:hadoop-common:2.10.2') { exclude group: 'org.slf4j', module: 'slf4j-api' exclude group: 'com.google.guava', module: 'guava' exclude group: 'commons-beanutils', module: 'commons-beanutils-core' @@ -55,7 +55,7 @@ dependencies { testImplementation "org.scalatest:scalatest_2.13:3.1.1" testImplementation 'org.apache.parquet:parquet-avro:1.11.0' testImplementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-avro:2.14.0' - testImplementation 'org.apache.hadoop:hadoop-client:2.9.2' + testImplementation 'org.apache.hadoop:hadoop-client:2.10.2' } spotless { diff --git a/gradle/dependency-locks/embulkPluginRuntime.lockfile b/gradle/dependency-locks/embulkPluginRuntime.lockfile index bd07ffa..ca7a748 100644 --- a/gradle/dependency-locks/embulkPluginRuntime.lockfile +++ b/gradle/dependency-locks/embulkPluginRuntime.lockfile @@ -2,6 +2,7 @@ # Manual edits can break the build and are not advised. # This file is expected to be part of source control. asm:asm:3.1 +ch.qos.reload4j:reload4j:1.2.19 com.amazonaws:aws-java-sdk-core:1.11.769 com.amazonaws:aws-java-sdk-glue:1.11.769 com.amazonaws:aws-java-sdk-kms:1.11.769 @@ -9,31 +10,32 @@ com.amazonaws:aws-java-sdk-s3:1.11.769 com.amazonaws:aws-java-sdk-sts:1.11.769 com.amazonaws:jmespath-java:1.11.769 com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.6.7 -com.fasterxml.woodstox:woodstox-core:5.0.3 +com.fasterxml.woodstox:woodstox-core:5.3.0 +com.github.spotbugs:spotbugs-annotations:3.1.9 com.github.stephenc.jcip:jcip-annotations:1.0-1 -com.google.code.findbugs:jsr305:3.0.0 +com.google.code.findbugs:jsr305:3.0.2 com.google.code.gson:gson:2.2.4 com.google.protobuf:protobuf-java:2.5.0 com.jamesmurty.utils:java-xmlbuilder:0.4 -com.jcraft:jsch:0.1.54 -com.nimbusds:nimbus-jose-jwt:4.41.1 +com.jcraft:jsch:0.1.55 +com.nimbusds:nimbus-jose-jwt:7.9 com.sun.jersey:jersey-core:1.9 com.sun.jersey:jersey-json:1.9 com.sun.jersey:jersey-server:1.9 com.sun.xml.bind:jaxb-impl:2.2.3-1 com.thoughtworks.paranamer:paranamer:2.3 -commons-beanutils:commons-beanutils:1.7.0 +commons-beanutils:commons-beanutils:1.9.4 commons-cli:commons-cli:1.2 commons-codec:commons-codec:1.11 commons-collections:commons-collections:3.2.2 commons-configuration:commons-configuration:1.6 commons-digester:commons-digester:1.8 -commons-io:commons-io:2.4 +commons-io:commons-io:2.5 commons-lang:commons-lang:2.6 commons-logging:commons-logging:1.2 commons-net:commons-net:3.1 commons-pool:commons-pool:1.6 -io.netty:netty:3.7.0.Final +io.netty:netty:3.10.6.Final javax.activation:activation:1.1 javax.annotation:javax.annotation-api:1.3.2 javax.servlet.jsp:jsp-api:2.1 @@ -46,21 +48,21 @@ net.java.dev.jets3t:jets3t:0.9.0 net.minidev:accessors-smart:1.2 net.minidev:json-smart:2.3 org.apache.avro:avro:1.7.7 -org.apache.commons:commons-compress:1.4.1 +org.apache.commons:commons-compress:1.21 org.apache.commons:commons-math3:3.1.1 -org.apache.curator:curator-client:2.7.1 -org.apache.curator:curator-framework:2.7.1 -org.apache.curator:curator-recipes:2.7.1 +org.apache.curator:curator-client:2.13.0 +org.apache.curator:curator-framework:2.13.0 +org.apache.curator:curator-recipes:2.13.0 org.apache.directory.api:api-asn1-api:1.0.0-M20 org.apache.directory.api:api-util:1.0.0-M20 org.apache.directory.server:apacheds-i18n:2.0.0-M15 org.apache.directory.server:apacheds-kerberos-codec:2.0.0-M15 -org.apache.hadoop:hadoop-annotations:2.9.2 -org.apache.hadoop:hadoop-auth:2.9.2 -org.apache.hadoop:hadoop-common:2.9.2 +org.apache.hadoop:hadoop-annotations:2.10.2 +org.apache.hadoop:hadoop-auth:2.10.2 +org.apache.hadoop:hadoop-common:2.10.2 org.apache.htrace:htrace-core4:4.1.0-incubating -org.apache.httpcomponents:httpclient:4.5.9 -org.apache.httpcomponents:httpcore:4.4.11 +org.apache.httpcomponents:httpclient:4.5.13 +org.apache.httpcomponents:httpcore:4.4.13 org.apache.parquet:parquet-column:1.11.0 org.apache.parquet:parquet-common:1.11.0 org.apache.parquet:parquet-encoding:1.11.0 @@ -69,13 +71,13 @@ org.apache.parquet:parquet-format:2.7.0 org.apache.parquet:parquet-hadoop:1.11.0 org.apache.parquet:parquet-jackson:1.11.0 org.apache.yetus:audience-annotations:0.11.0 -org.apache.zookeeper:zookeeper:3.4.6 +org.apache.zookeeper:zookeeper:3.4.14 org.codehaus.jackson:jackson-core-asl:1.9.13 org.codehaus.jackson:jackson-jaxrs:1.8.3 org.codehaus.jackson:jackson-mapper-asl:1.9.13 org.codehaus.jackson:jackson-xc:1.8.3 org.codehaus.jettison:jettison:1.1 -org.codehaus.woodstox:stax2-api:3.1.4 +org.codehaus.woodstox:stax2-api:4.2.1 org.mortbay.jetty:jetty-sslengine:6.1.26 org.mortbay.jetty:jetty-util:6.1.26 org.mortbay.jetty:jetty:6.1.26 @@ -83,7 +85,7 @@ org.mortbay.jetty:servlet-api:2.5-20081211 org.ow2.asm:asm:5.0.4 org.scala-lang:scala-library:2.13.1 org.slf4j:slf4j-log4j12:1.7.25 -org.tukaani:xz:1.0 +org.slf4j:slf4j-reload4j:1.7.36 org.xerial.snappy:snappy-java:1.1.7.3 software.amazon.ion:ion-java:1.0.2 xmlenc:xmlenc:0.52