use overwrite mode

Signed-off-by: Tobias Windisch <tobias.windisch@hs-kempten.de>
boschglobal · Dec 13, 2023 · c833438 · c833438
1 parent 5bdb0cf
commit c833438
Showing 1 changed file with 1 addition and 3 deletions.
diff --git a/luisy/targets.py b/luisy/targets.py
@@ -252,10 +252,8 @@ def write(self, df: SparkDataFrame):
         Args:
             df (pyspark.sql.DataFrame): Dataframe that should be written to delta table
         """
-        logger.info(f"Drop table {self.table_uri}")
-        self.spark.sql(f"DROP TABLE IF EXISTS {self.table_uri}")
         logger.info(f"Write to {self.table_uri}")
-        df.write.saveAsTable(self.table_uri)
+        df.write.mode("overwrite").saveAsTable(self.table_uri)
 
     def read(self):
         return self.spark.table(self.table_uri)