From c8334385fa4dc6d20ea9c7cd734c2c1972ec44df Mon Sep 17 00:00:00 2001 From: Tobias Windisch Date: Wed, 13 Dec 2023 23:19:21 +0100 Subject: [PATCH] use overwrite mode Signed-off-by: Tobias Windisch --- luisy/targets.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/luisy/targets.py b/luisy/targets.py index f936122..54269e1 100644 --- a/luisy/targets.py +++ b/luisy/targets.py @@ -252,10 +252,8 @@ def write(self, df: SparkDataFrame): Args: df (pyspark.sql.DataFrame): Dataframe that should be written to delta table """ - logger.info(f"Drop table {self.table_uri}") - self.spark.sql(f"DROP TABLE IF EXISTS {self.table_uri}") logger.info(f"Write to {self.table_uri}") - df.write.saveAsTable(self.table_uri) + df.write.mode("overwrite").saveAsTable(self.table_uri) def read(self): return self.spark.table(self.table_uri)