From 0b56e2ba1786b7bdc1124d50ff7ab7725b913046 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pascal=20Spo=CC=88rri?= Date: Tue, 19 Sep 2023 13:17:34 +0200 Subject: [PATCH] S3A: Use an array-backed buffer for uploading. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Pascal SpoĢˆrri --- examples/sql/run_benchmark.sh | 4 +--- examples/terasort/run.sh | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/examples/sql/run_benchmark.sh b/examples/sql/run_benchmark.sh index c97b48f..f70ff96 100755 --- a/examples/sql/run_benchmark.sh +++ b/examples/sql/run_benchmark.sh @@ -51,12 +51,10 @@ SPARK_HADOOP_S3A_CONFIG=( --conf spark.hadoop.fs.s3a.path.style.access=true --conf spark.hadoop.fs.s3a.fast.upload=true --conf spark.hadoop.fs.s3a.block.size=$(($BLOCK_SIZE * 1024 * 1024)) + --conf spark.hadoop.fs.s3a.fast.upload.buffer=array ) SPARK_S3_SHUFFLE_CONFIG=( - --conf spark.hadoop.fs.s3a.access.key=${S3A_ACCESS_KEY} - --conf spark.hadoop.fs.s3a.secret.key=${S3A_SECRET_KEY} - --conf spark.hadoop.fs.s3a.endpoint=${S3A_ENDPOINT} --conf spark.shuffle.manager="org.apache.spark.shuffle.sort.S3ShuffleManager" --conf spark.shuffle.sort.io.plugin.class=org.apache.spark.shuffle.S3ShuffleDataIO --conf spark.shuffle.checksum.enabled=${CHECKSUM_ENABLED} diff --git a/examples/terasort/run.sh b/examples/terasort/run.sh index 19c055f..f91454b 100755 --- a/examples/terasort/run.sh +++ b/examples/terasort/run.sh @@ -52,13 +52,11 @@ SPARK_HADOOP_S3A_CONFIG=( --conf spark.hadoop.fs.s3a.path.style.access=true --conf spark.hadoop.fs.s3a.fast.upload=true --conf spark.hadoop.fs.s3a.block.size=$(($BLOCK_SIZE * 1024 * 1024)) + --conf spark.hadoop.fs.s3a.fast.upload.buffer=array ) SPARK_S3_SHUFFLE_CONFIG=( - --conf spark.hadoop.fs.s3a.access.key=${S3A_ACCESS_KEY} - --conf spark.hadoop.fs.s3a.secret.key=${S3A_SECRET_KEY} - --conf spark.hadoop.fs.s3a.endpoint=${S3A_ENDPOINT} --conf spark.shuffle.manager="org.apache.spark.shuffle.sort.S3ShuffleManager" --conf spark.shuffle.sort.io.plugin.class=org.apache.spark.shuffle.S3ShuffleDataIO --conf spark.shuffle.checksum.enabled=${CHECKSUM_ENABLED}