etl

nogibjj · Dec 13, 2024 · 10522b8 · 10522b8
1 parent 2230918
commit 10522b8
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 7 deletions.
diff --git a/databricks_scripts/extract.py b/databricks_scripts/extract.py
@@ -1,15 +1,21 @@
-# from my_lib.util import log_tests
-# from my_lib.extract import extract
-%run ../my_lib/util
-%run ../my_lib/extract
+import sys
+
+sys.path.append("/Workspace/Workspace/Shared/Leonard_Eshun_Mini_Project_Eleven/my_lib/")
+
+from util import log_tests
+from extract_data import extract
 import os
 
 
 def do_extract():
     log_tests("Extraction Test", header=True, new_log_file=True)
     log_tests("Removing existing CSV file exists")
-    if os.path.exists("/tmp/data/air_quality.csv"):
-        os.remove("/tmp/data/air_quality.csv")
+    if os.path.exists(
+        "/Workspace/Workspace/Shared/Leonard_Eshun_Mini_Project_Eleven/data/air_quality.csv"
+    ):
+        os.remove(
+            "/Workspace/Workspace/Shared/Leonard_Eshun_Mini_Project_Eleven/data/air_quality.csv"
+        )
 
     log_tests("Confirming that CSV file doesn't exists...")
     assert not os.path.exists("population_bar.png")
@@ -22,7 +28,9 @@ def do_extract():
     )
 
     log_tests("Testing if CSV file exists...")
-    assert os.path.exists("/tmp/data/air_quality.csv")
+    assert os.path.exists(
+        "/Workspace/Workspace/Shared/Leonard_Eshun_Mini_Project_Eleven/data/air_quality.csv"
+    )
     log_tests("Extraction Test Successful", last_in_group=True)
     print("Extraction Test Successful")
 

diff --git a/my_lib/extract.py → my_lib/extract_data.py b/my_lib/extract.py → my_lib/extract_data.py