diff --git a/bgcArgoDMQC/core/core.py b/bgcArgoDMQC/core/core.py index 98afe1a..c985bbd 100644 --- a/bgcArgoDMQC/core/core.py +++ b/bgcArgoDMQC/core/core.py @@ -269,6 +269,20 @@ def read_flat_variables(nc): floatData = dict() for name, dim in nc.dimensions.items(): floatData[name] = dim.size + + n_prof = nc.dimensions['N_PROF'].size + n_levels = nc.dimensions['N_LEVELS'].size + n_prof_array = [] + n_levels_array = [] + for i in range(n_prof): + n_prof_array = n_prof_array + n_levels*[i] + n_levels_array = n_levels_array + list(range(n_levels)) + + floatData['N_PROF_DIM'] = floatData['N_PROF'] + floatData['N_LEVELS_DIM'] = floatData['N_LEVELS'] + floatData['N_PROF'] = np.array(n_prof_array) + floatData['N_LEVELS'] = np.array(n_levels_array) + for name, var in nc.variables.items(): floatData[name] = var[:].data.flatten() diff --git a/bgcArgoDMQC/core/prof.py b/bgcArgoDMQC/core/prof.py index ab6f8f7..dbf9ea3 100644 --- a/bgcArgoDMQC/core/prof.py +++ b/bgcArgoDMQC/core/prof.py @@ -156,7 +156,8 @@ def to_dataframe(self): ''' df = pd.DataFrame() - n_level = self.__floatdict__['N_LEVELS'] + n_level = self.__floatdict__['N_LEVELS_DIM'] + n_prof = self.__floatdict__['N_PROF_DIM'] priority_vars = ['PRES', 'PRES_QC', 'TEMP', 'TEMP_QC', 'PSAL', 'PSAL_QC'] if self.kind in ['C', 'S'] else ['PRES'] bgc_vars = list(set(self.__floatdict__.keys()) & set(['DOXY', 'DOXY_QC', 'DOXY_ADJUSTED', 'DOXY_ADJUSTED_QC', 'CHLA', 'CHLA_QC', 'CHLA_ADJUSTED', 'CHLA_ADJUSTED_QC', 'BBP700', 'BBP700_QC', 'BBP700_ADJUSTED', 'BBP_ADJUSTED_QC'])) priority_vars = priority_vars + bgc_vars @@ -166,9 +167,11 @@ def to_dataframe(self): for k in set(self.__floatdict__.keys()) - set(df.columns): dim = self.__floatdict__[k].shape[0] if type(self.__floatdict__[k]) is np.ndarray else np.inf - if dim == n_level: + if dim == n_level*n_prof: df[k] = self.__floatdict__[k] + df = df.set_index(['N_PROF', 'N_LEVELS']) + self.df = df return copy.deepcopy(self.df) diff --git a/bgcArgoDMQC/io/netcdf.py b/bgcArgoDMQC/io/netcdf.py index b0ae324..96dc351 100644 --- a/bgcArgoDMQC/io/netcdf.py +++ b/bgcArgoDMQC/io/netcdf.py @@ -355,6 +355,25 @@ def export_delayed_files(fdict, files, gain, data_mode='D', comment=None, equati sys.stdout.write('done\n') D_nc.close() +def reshape(data, n_prof, n_levels): + + print(data) + out = np.nan*np.ones((n_prof, n_levels)) + for i in range(n_prof): + out[i,:] = data[i*n_levels:(i+1)*n_levels] + print(out) + return out + +def find_param(nc, param): + + for i in range(nc.dimensions['N_PROF'].size): + param_list = [read_ncstr(a) for a in nc['PARAMETER'][:].data[i,0,:,:]] + if param in param_list: + param_index = param_list.index(param) + break + + return (i, param_index) + def update_nc(fdict, fn, changelog, history_dict={}): # get DATE_UPDATE @@ -372,12 +391,11 @@ def update_nc(fdict, fn, changelog, history_dict={}): O_nc = unlimit_dimension(fn, output_file, 'N_HISTORY') for f in changelog: - - O_nc[f][:] = fdict[f] + O_nc[f][:] = reshape(fdict[f], fdict['N_PROF_DIM'], fdict['N_LEVELS_DIM']) if fdict['N_PROF_DIM'] > 1 else fdict[f] # find index along PARAMETER - param_index = [read_ncstr(a) for a in O_nc['PARAMETER'][:].data[0,0,:,:]].index(f.replace('_QC','').replace('_ADJUSTED','')) - data_mode = O_nc['PARAMETER_DATA_MODE'][:][0,param_index].decode() + param_index = find_param(O_nc, f.replace('_QC','').replace('_ADJUSTED','')) + data_mode = O_nc['PARAMETER_DATA_MODE'][:][param_index].decode() # if a changed value is QC flags, recalculate PROFILE__QC if f.split('_')[-1] == 'QC':