From 5457992f236085832ab219be8ed8cfeaabf9ad19 Mon Sep 17 00:00:00 2001 From: Leonard Eshun Date: Fri, 13 Dec 2024 23:26:20 +0000 Subject: [PATCH] etl --- my_lib/load.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/my_lib/load.py b/my_lib/load.py index e76613c..4865567 100644 --- a/my_lib/load.py +++ b/my_lib/load.py @@ -121,6 +121,17 @@ def transform_n_load( spark.createDataFrame(value, static_data.spark_dataframes[key].columns) ) + # save data + static_data.spark_dataframes["indicator"].write.format("delta").mode( + "overwrite" + ).saveAsTable("indicator") + static_data.spark_dataframes["geo_data"].write.format("delta").mode( + "overwrite" + ).saveAsTable("geo_data") + static_data.spark_dataframes["air_quality"].write.format("delta").mode( + "overwrite" + ).saveAsTable("air_quality") + static_data.spark_dataframes["indicator"].show() static_data.spark_dataframes["geo_data"].show() static_data.spark_dataframes["air_quality"].show()