Merge pull request #72 from karnesh/main

Added functionality to generate csv files for Tethys app

Model/Neural_Network/sweml_hindcast.py

@@ -17,7 +17,7 @@
 
 def sweml_hindcast(new_year, threshold, Region_list, fSCA, frequency, NewSim, single_day):
     model = 'Neural_Network'
-
+    
     retries = 0
     if single_day:
         prev_data_flag = True
@@ -38,13 +38,14 @@ def sweml_hindcast(new_year, threshold, Region_list, fSCA, frequency, NewSim, si
     else:
         datelist = Hindcast_Initialization.Hindcast_Initialization(cwd, datapath, new_year, threshold, Region_list,
                                                                    frequency, fSCA=fSCA)
-
+
+
     # Run data processing script to partition key regional dataframes
     # note, need to load RegionTrain_SCA.h5,
     if datelist[0][-5:] == '10-01':
         RegionTrain, RegionTest, RegionObs_Train, RegionObs_Test, RegionTest_notScaled = DataProcess.DataProcess(
             new_year, frequency, model, Region_list, fSCA=fSCA)
-
+        
         """
         # model training, each participants model will be different but should follow the prescribed input feature 
         template epochs= 30 
@@ -79,13 +80,20 @@ def sweml_hindcast(new_year, threshold, Region_list, fSCA, frequency, NewSim, si
         snow.SWE_Predict(NewSim=NewSim, Corrections=False, fSCA=fSCA)
         snow.netCDF_compressed(plot=False)
         snow.Geo_df()
-    
+
     modelname = 'Neural_Network'
-    folderpath = 'Predictions/Hold_Out_Year/Daily/fSCA_True/'
-    AWSpath = f"Hold_Out_Year/Daily/"
-    file_types = ['.h5', '.pkl']
+    folderpath = ['Predictions/Hold_Out_Year/Daily/fSCA_True/', f'{datapath}/data/csv/', f'{datapath}/data/GeoJSON/']
+    AWSpath = [f"Hold_Out_Year/Daily/", f"Hold_Out_Year/Daily/csv/", f"Hold_Out_Year/Daily/GeoJSON/"]
+    file_types = ['.h5', '.pkl', '.csv', '.geojson']
     for file_type in file_types:
-        Hindcast_Initialization.Hindcast_to_AWS(modelname, folderpath, AWSpath, file_type)
+        if file_type == '.h5' or file_type == '.pkl':
+            index = 0
+            continue
+        elif file_type == '.csv':
+            index = 1
+        else:
+            index = 2
+        Hindcast_Initialization.Hindcast_to_AWS(modelname, folderpath[index], AWSpath[index], file_type)
 
     if retries > 0:
         sweml_hindcast(new_year, threshold, Region_list, fSCA, frequency, NewSim, single_day)

Model/shared_scripts/National_Snow_Model.py

@@ -58,6 +58,7 @@
 from osgeo import gdal, osr
 from matplotlib.dates import date2num
 from rasterio.crs import CRS
+import csv
 
 # import contextily as ctx
 import ulmo
@@ -1285,7 +1286,7 @@ def netCDF_compressed(self, plot):
 
         # create nc filepath
         #fn = self.cwd + '/Data/NetCDF/SWE_' + self.date + '_compressed.nc'
-        fn = f"{HOME}/SWEML/Data/NetCDF/SWE_{self.date}_compressed.nc"
+        fn = f"{HOME}/SWEML/data/NetCDF/SWE_{self.date}_compressed.nc"
 
         # make nc file, set lat/long, time
         ncfile = nc.Dataset(fn, 'w', format='NETCDF4')
@@ -1554,7 +1555,7 @@ def plot_interactive_SWE(self, pinlat, pinlong, web):
     def Geo_df(self):
         print('loading file')
         #fnConus = self.cwd + '/Data/NetCDF/SWE_' + self.date + '_compressed.nc'
-        fnConus = f"{HOME}/SWEML/Data/NetCDF/SWE_{self.date}_compressed.nc"
+        fnConus = f"{HOME}/SWEML/data/NetCDF/SWE_{self.date}_compressed.nc"
         # requires the netCDF4 package rather than rioxarray
         xrConus = nc.Dataset(fnConus)
 
@@ -1568,6 +1569,7 @@ def Geo_df(self):
         SWE_pd = pd.DataFrame.from_dict({'SWE': SWE, 'x': x, 'y': y})
         SWE_threshold = 0.1
         SWE_pd = SWE_pd[SWE_pd['SWE'] > SWE_threshold]
+        SWE_pd['SWE'] = SWE_pd['SWE'].round(2)
         SWE_gdf = gpd.GeoDataFrame(
             SWE_pd, geometry=gpd.points_from_xy(SWE_pd.x, SWE_pd.y), crs=4326)
 
@@ -1579,8 +1581,16 @@ def Geo_df(self):
         Chorocols = ['geoid', 'SWE', 'geometry']
         self.SWE_gdf = SWE_gdf[Chorocols]
         self.SWE_gdf.crs = CRS.from_epsg(4326)
-        file = f"{HOME}/SWEML/Data/GeoJSON/SWE_{self.date}.geojson"
+        file = f"{HOME}/SWEML/data/GeoJSON/SWE_{self.date}.geojson"
         SWE_gdf.to_file(file, driver='GeoJSON')
+        for index, row in SWE_gdf.iterrows():
+            csv_file = f"{HOME}/SWEML/data/csv/swe_1000m_{row['y']:.3f}_{row['x']:.3f}.csv"
+            file_exists = os.path.isfile(csv_file)
+            with open(csv_file, 'a') as f:
+                writer = csv.writer(f)
+                if not file_exists:
+                    writer.writerow(['date', 'SWE'])
+                writer.writerow([self.date, row['SWE']])
         xrConus.close()
 
     # produce an interactive plot using Folium