Update medium-stats-spark-driver.py

pavan-kumar-99 · Jun 10, 2024 · 7526e63 · 7526e63
1 parent fd08c77
commit 7526e63
Showing 1 changed file with 5 additions and 1 deletion.
diff --git a/scripts/medium/medium-stats-spark-driver.py b/scripts/medium/medium-stats-spark-driver.py
@@ -56,7 +56,7 @@ def process_data(bucket_name, bucket_prefix, local_path):
     catalog_name = "glue_catalog"
     iceberg_bucket_name = bucket_name
     iceberg_bucket_prefix = bucket_prefix
-    warehouse_path = f"s3://{iceberg_bucket_name}/{iceberg_bucket_prefix}"
+    warehouse_path = f"s3a://{iceberg_bucket_name}/{iceberg_bucket_prefix}"
     mongodb_uri = os.getenv("MONGO_URI")
 
     if not mongodb_uri:
@@ -82,6 +82,10 @@ def process_data(bucket_name, bucket_prefix, local_path):
             f"spark.sql.catalog.{catalog_name}.s3.endpoint",
             "http://minio-server.minio:9000",
         )
+        .config(
+            f"spark.sql.catalog.{catalog_name}.s3a.endpoint",
+            "http://minio-server.minio:9000",
+        )
         .config(
             f"spark.sql.catalog.{catalog_name}.io-impl",
             "org.apache.iceberg.aws.s3.S3FileIO",