Update medium-stats-spark-driver.py

pavan-kumar-99 · Jun 12, 2024 · 7dba05b · 7dba05b
1 parent 6ab67d7
commit 7dba05b
Showing 1 changed file with 2 additions and 6 deletions.
diff --git a/scripts/medium/medium-stats-spark-driver.py b/scripts/medium/medium-stats-spark-driver.py
@@ -78,13 +78,9 @@ def process_data(bucket_name, bucket_prefix, local_path):
             "hadoop",
         )
         .config(f"spark.sql.defaultCatalog", f"{catalog_name}")
-        .config(
-            f"spark.sql.catalog.{catalog_name}.s3.endpoint",
-            "http://minio-server.minio:9000",
-        )
         .config(
             f"spark.sql.catalog.{catalog_name}.s3a.endpoint",
-            "http://minio-server.minio:9000",
+            "http://minio-server.minio:9000/",
         )
         .config(
             "spark.sql.extensions",
@@ -208,7 +204,7 @@ def compute_yearly_statistics(spark, table_name):
     local_path = "/usr/stats/" + args.key
     bucket_name = "medium-stats"
     iceberg_bucket_name = "iceberg-table"
-    iceberg_bucket_prefix = "iceberg-tables/"
+    iceberg_bucket_prefix = "warehouse/"
     temp_table_name = "mediumstatstemp"
 
     download_from_s3(bucket_name, args.key, local_path)