changes to prof file export

- handles N_PROF > 1 better - imporved param index finding
ArgoCanada · Jul 17, 2024 · 70dff91 · 70dff91
1 parent 88a150a
commit 70dff91
Show file tree

Hide file tree

Showing 3 changed files with 41 additions and 6 deletions.
diff --git a/bgcArgoDMQC/core/core.py b/bgcArgoDMQC/core/core.py
@@ -269,6 +269,20 @@ def read_flat_variables(nc):
     floatData = dict()
     for name, dim in nc.dimensions.items():
         floatData[name] = dim.size
+
+    n_prof = nc.dimensions['N_PROF'].size
+    n_levels = nc.dimensions['N_LEVELS'].size
+    n_prof_array = []
+    n_levels_array = []
+    for i in range(n_prof):
+        n_prof_array = n_prof_array + n_levels*[i]
+        n_levels_array = n_levels_array + list(range(n_levels))
+
+    floatData['N_PROF_DIM'] = floatData['N_PROF']
+    floatData['N_LEVELS_DIM'] = floatData['N_LEVELS']
+    floatData['N_PROF'] = np.array(n_prof_array)
+    floatData['N_LEVELS'] = np.array(n_levels_array)
+
     for name, var in nc.variables.items():
         floatData[name] = var[:].data.flatten()
 

diff --git a/bgcArgoDMQC/core/prof.py b/bgcArgoDMQC/core/prof.py
@@ -156,7 +156,8 @@ def to_dataframe(self):
         '''
 
         df = pd.DataFrame()
-        n_level = self.__floatdict__['N_LEVELS']
+        n_level = self.__floatdict__['N_LEVELS_DIM']
+        n_prof = self.__floatdict__['N_PROF_DIM']
         priority_vars = ['PRES', 'PRES_QC', 'TEMP', 'TEMP_QC', 'PSAL', 'PSAL_QC'] if self.kind in ['C', 'S'] else ['PRES']
         bgc_vars = list(set(self.__floatdict__.keys()) & set(['DOXY', 'DOXY_QC', 'DOXY_ADJUSTED', 'DOXY_ADJUSTED_QC', 'CHLA', 'CHLA_QC', 'CHLA_ADJUSTED', 'CHLA_ADJUSTED_QC', 'BBP700', 'BBP700_QC', 'BBP700_ADJUSTED', 'BBP_ADJUSTED_QC']))
         priority_vars = priority_vars + bgc_vars
@@ -166,9 +167,11 @@ def to_dataframe(self):
 
         for k in set(self.__floatdict__.keys()) - set(df.columns):
             dim = self.__floatdict__[k].shape[0] if type(self.__floatdict__[k]) is np.ndarray else np.inf
-            if dim == n_level:
+            if dim == n_level*n_prof:
                 df[k] = self.__floatdict__[k]
 
+        df = df.set_index(['N_PROF', 'N_LEVELS'])
+
         self.df = df
         return copy.deepcopy(self.df)
 

diff --git a/bgcArgoDMQC/io/netcdf.py b/bgcArgoDMQC/io/netcdf.py
@@ -355,6 +355,25 @@ def export_delayed_files(fdict, files, gain, data_mode='D', comment=None, equati
         sys.stdout.write('done\n')
         D_nc.close()
 
+def reshape(data, n_prof, n_levels):
+
+    print(data)
+    out = np.nan*np.ones((n_prof, n_levels))
+    for i in range(n_prof):
+        out[i,:] = data[i*n_levels:(i+1)*n_levels]
+    print(out)
+    return out
+
+def find_param(nc, param):
+
+    for i in range(nc.dimensions['N_PROF'].size):
+        param_list = [read_ncstr(a) for a in nc['PARAMETER'][:].data[i,0,:,:]]
+        if param in param_list:
+            param_index = param_list.index(param)
+            break
+
+    return (i, param_index)
+
 def update_nc(fdict, fn, changelog, history_dict={}):
 
     # get DATE_UPDATE
@@ -372,12 +391,11 @@ def update_nc(fdict, fn, changelog, history_dict={}):
         O_nc = unlimit_dimension(fn, output_file, 'N_HISTORY')
 
     for f in changelog:
-
-        O_nc[f][:] = fdict[f]
+        O_nc[f][:] = reshape(fdict[f], fdict['N_PROF_DIM'], fdict['N_LEVELS_DIM']) if fdict['N_PROF_DIM'] > 1 else fdict[f]
 
         # find index along PARAMETER
-        param_index = [read_ncstr(a) for a in O_nc['PARAMETER'][:].data[0,0,:,:]].index(f.replace('_QC','').replace('_ADJUSTED',''))
-        data_mode = O_nc['PARAMETER_DATA_MODE'][:][0,param_index].decode()
+        param_index = find_param(O_nc, f.replace('_QC','').replace('_ADJUSTED',''))
+        data_mode = O_nc['PARAMETER_DATA_MODE'][:][param_index].decode()
 
         # if a changed value is QC flags, recalculate PROFILE_<PARAM>_QC
         if f.split('_')[-1] == 'QC':