diff --git a/airflow/dags/Bugs_DAG.py b/airflow/dags/Bugs_DAG.py index 3d69fcd..a597e33 100644 --- a/airflow/dags/Bugs_DAG.py +++ b/airflow/dags/Bugs_DAG.py @@ -17,7 +17,7 @@ S3_BUCKET = "de5-s4tify" -S3_CSV_KEY = f"raw_data/bugs_chart_data/bugs_chart_with_{TODAY}.csv" +S3_CSV_KEY = f"raw_data/bugs_chart_data/bugs_chart_{TODAY}.csv" LOCAL_FILE_PATH = f"/opt/airflow/data/bugs_chart_with_genre_{TODAY}.csv" diff --git a/airflow/dags/scripts/S3_Spark_SnowFlake_ELT.py b/airflow/dags/scripts/S3_Spark_SnowFlake_ELT.py index 9c656ff..e5a54ba 100644 --- a/airflow/dags/scripts/S3_Spark_SnowFlake_ELT.py +++ b/airflow/dags/scripts/S3_Spark_SnowFlake_ELT.py @@ -171,11 +171,11 @@ def insert_data_into_snowflake(df, table_name): TODAY = datetime.now().strftime("%Y%m%d") S3_BUCKET = "s3a://de5-s4tify" chart_sources = { - "bugs": f"{S3_BUCKET}/raw_data/bugs_chart/bugs_chart_{TODAY}.csv", - "flo": f"{S3_BUCKET}/raw_data/flo_chart/flo_chart_{TODAY}.csv", - "genie": f"{S3_BUCKET}/raw_data/genie_chart/genie_chart_{TODAY}.csv", - "melon": f"{S3_BUCKET}/raw_data/melon_chart/melon_chart_{TODAY}.csv", - "vibe": f"{S3_BUCKET}/raw_data/vibe_chart/vibe_chart_{TODAY}.csv", + "bugs": f"{S3_BUCKET}/raw_data/bugs_chart_data/bugs_chart_{TODAY}.csv", + "flo": f"{S3_BUCKET}/raw_data/flo_chart_data/flo_chart_{TODAY}.csv", + "genie": f"{S3_BUCKET}/raw_data/genie_chart_data/genie_chart_{TODAY}.csv", + "melon": f"{S3_BUCKET}/raw_data/melon_chart_data/melon_chart_{TODAY}.csv", + "vibe": f"{S3_BUCKET}/raw_data/vibe_chart_data/vibe_chart_{TODAY}.csv", }