fix main function args logic

Author: Naman Jain

Dockerfile

@@ -12,4 +12,4 @@ COPY . .
 
 ENV PYTHONPATH=/app
 
-ENTRYPOINT ["poetry", "run", "python", "src/__main__.py"]
+ENTRYPOINT ["poetry", "run", "python", "netcdf_to_parquet/__main__.py"]

README.md

@@ -14,19 +14,19 @@ The transformed data supports:
 ### Using Docker
 To build the docker image, run:
 ```sh
-docker build -t data_transformations .
+docker build -t netcdf_to_parquet .
 ```
 
 To run the docker container, here's an example command:
-`docker run -v $(pwd)/output:/app/output data_transformations <start_date> <end_date> <out_dir>`
+`docker run -v $(pwd)/output:/app/output netcdf_to_parquet <start_date> <end_date> <out_dir>`
 <br/>
 Here, `start_date` and `end_date` are in DD-MM-YYYY format. 
 <br/>
 And `out_dir` is the directory where the parquet files will be generated. 
 
 For example:
 ```sh
-docker run -v $(pwd)/output:/app/output data_transformations 01-01-2022 04-01-2022 out_dir
+docker run -v $(pwd)/output:/app/output netcdf_to_parquet 01-01-2022 04-01-2022 out_dir
 ```
 
 ### Using Poetry
@@ -36,7 +36,7 @@ To generate and activate the environment, run following commands from the root d
 ```sh
 poetry install
 poetry shell
-python -m data_transformations 01-01-2023 03-01-2023 ./parquet_files
+python -m netcdf_to_parquet 01-01-2023 03-01-2023 ./parquet_files
 ```
 
 ### Using pip
@@ -116,6 +116,7 @@ Note that some of the tests are live tests that actually download a file from GC
  - add coverage report?
  - fix black and flake8 conflict
  - why is GCS getting initialized when imported as library
+ - add a test for using it as a library
 
 ## Improvements
  - speed up

netcdf_to_parquet/__main__.py

@@ -80,37 +80,28 @@ def main(
         out_dir (str): Output directory for Parquet files. Defaults to "parquet_files".
         args (argparse.Namespace, optional): Command-line arguments. Defaults to None.
     """
-    if start_date is None or end_date is None:
-        if args is None:
-            raise ValueError(
-                "Please provide values for start_date and end_date either as arguments or through the command line interface."
-            )
+    if args is None:
+        args = parse_arguments()
 
-        start_date = args.start_date or start_date
-        end_date = args.end_date or end_date
-        out_dir = args.out_dir or out_dir
+    start_date = args.start_date or start_date
+    end_date = args.end_date or end_date
+    out_dir = args.out_dir or out_dir
 
-    # TODO: improve logic
     if start_date is None or end_date is None:
         raise ValueError("start_date and end_date must be provided.")
 
     pathlib.Path(out_dir).mkdir(parents=True, exist_ok=True)
-
     current_date = start_date
     total_days = (end_date - current_date).days + 1
-
     for _ in tqdm.tqdm(range(total_days), desc="Processing dates"):
         try:
             date_str = current_date.strftime("%Y/%m/%d")
             file_path = f"{constants.GCS_BASE_URL}/{date_str}/total_precipitation/surface.nc"
             output_path = f"{out_dir}/precipitation_{current_date.strftime('%d_%m_%Y')}.parquet"
             netcdf_to_parquet(file_path, output_path, file_system)
         except Exception as e:
-            logging.error(
-                f"Failed to process date {current_date.strftime('%d_%m_%Y')}: {e}"
-            )
-        finally:
-            current_date += datetime.timedelta(days=1)
+            logging.error(f"Failed to process date {current_date}: {e}")
+        current_date += datetime.timedelta(days=1)
 
 
 if __name__ == "__main__":

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "netcdf-to-parquet"
-version = "0.1.4"
+version = "0.1.5"
 description = "Transforms total precipitation NetCDF files into Parquet format."
 authors = ["Naman Jain"]
 license = "MIT"