Merge pull request #408 from aodn/2.5.24

2.5.24
aodn · Jan 17, 2017 · c0f984a · c0f984a
2 parents 0c75724 + 5972701
commit c0f984a
Show file tree

Hide file tree

Showing 39 changed files with 1,134 additions and 882 deletions.
diff --git a/AutomaticQC/imosCorrMagVelocitySetQC.m b/AutomaticQC/imosCorrMagVelocitySetQC.m
@@ -111,7 +111,7 @@
 % read dataset QC parameters if exist and override previous 
 % parameters file
 currentQCtest = mfilename;
-cmag = readQCparameter(sample_data.toolbox_input_file, currentQCtest, 'cmag', cmag);
+cmag = readDatasetParameter(sample_data.toolbox_input_file, currentQCtest, 'cmag', cmag);
 
 paramsLog = ['cmag=' num2str(cmag)];
 
@@ -158,6 +158,6 @@
 end
 
 % write/update dataset QC parameters
-writeQCparameter(sample_data.toolbox_input_file, currentQCtest, 'cmag', cmag);
+writeDatasetParameter(sample_data.toolbox_input_file, currentQCtest, 'cmag', cmag);
 
 end
diff --git a/AutomaticQC/imosEchoIntensityVelocitySetQC.m b/AutomaticQC/imosEchoIntensityVelocitySetQC.m
@@ -116,7 +116,7 @@
 % read dataset QC parameters if exist and override previous 
 % parameters file
 currentQCtest = mfilename;
-ea_thresh = readQCparameter(sample_data.toolbox_input_file, currentQCtest, 'ea_thresh', ea_thresh);
+ea_thresh = readDatasetParameter(sample_data.toolbox_input_file, currentQCtest, 'ea_thresh', ea_thresh);
 
 paramsLog = ['ea_thresh=' num2str(ea_thresh)];
 
@@ -189,6 +189,6 @@
 end
 
 % write/update dataset QC parameters
-writeQCparameter(sample_data.toolbox_input_file, currentQCtest, 'ea_thresh', ea_thresh);
+writeDatasetParameter(sample_data.toolbox_input_file, currentQCtest, 'ea_thresh', ea_thresh);
 
 end
diff --git a/AutomaticQC/imosErrorVelocitySetQC.m b/AutomaticQC/imosErrorVelocitySetQC.m
@@ -97,7 +97,7 @@
 % read dataset QC parameters if exist and override previous 
 % parameters file
 currentQCtest = mfilename;
-err_vel = readQCparameter(sample_data.toolbox_input_file, currentQCtest, 'err_vel', err_vel);
+err_vel = readDatasetParameter(sample_data.toolbox_input_file, currentQCtest, 'err_vel', err_vel);
 
 paramsLog = ['err_vel=' num2str(err_vel)];
 
@@ -141,6 +141,6 @@
 end
 
 % write/update dataset QC parameters
-writeQCparameter(sample_data.toolbox_input_file, currentQCtest, 'err_vel', err_vel);
+writeDatasetParameter(sample_data.toolbox_input_file, currentQCtest, 'err_vel', err_vel);
 
 end
diff --git a/AutomaticQC/imosHorizontalVelocitySetQC.m b/AutomaticQC/imosHorizontalVelocitySetQC.m
@@ -93,7 +93,7 @@
 % read dataset QC parameters if exist and override previous 
 % parameters file
 currentQCtest = mfilename;
-hvel = readQCparameter(sample_data.toolbox_input_file, currentQCtest, 'hvel', hvel);
+hvel = readDatasetParameter(sample_data.toolbox_input_file, currentQCtest, 'hvel', hvel);
 
 paramsLog = ['hvel=' num2str(hvel)];
 
@@ -127,6 +127,6 @@
 end
 
 % write/update dataset QC parameters
-writeQCparameter(sample_data.toolbox_input_file, currentQCtest, 'hvel', hvel);
+writeDatasetParameter(sample_data.toolbox_input_file, currentQCtest, 'hvel', hvel);
 
 end
diff --git a/AutomaticQC/imosImpossibleDateQC.m b/AutomaticQC/imosImpossibleDateQC.m
@@ -101,8 +101,8 @@
     % read dataset QC parameters if exist and override previous 
     % parameters file
     currentQCtest = mfilename;
-    dateMin = readQCparameter(sample_data.toolbox_input_file, currentQCtest, 'dateMin', dateMin);
-    dateMax = readQCparameter(sample_data.toolbox_input_file, currentQCtest, 'dateMax', dateMax);
+    dateMin = readDatasetParameter(sample_data.toolbox_input_file, currentQCtest, 'dateMin', dateMin);
+    dateMax = readDatasetParameter(sample_data.toolbox_input_file, currentQCtest, 'dateMax', dateMax);
 
     paramsLog = ['dateMin=' datestr(dateMin, 'dd/mm/yyyy') ...
         ', dateMax=' datestr(dateMax, 'dd/mm/yyyy')];
@@ -122,6 +122,6 @@
     end
 
     % write/update dataset QC parameters
-    writeQCparameter(sample_data.toolbox_input_file, currentQCtest, 'dateMin', dateMin);
-    writeQCparameter(sample_data.toolbox_input_file, currentQCtest, 'dateMax', dateMax);
+    writeDatasetParameter(sample_data.toolbox_input_file, currentQCtest, 'dateMin', dateMin);
+    writeDatasetParameter(sample_data.toolbox_input_file, currentQCtest, 'dateMax', dateMax);
 end
diff --git a/AutomaticQC/imosImpossibleDepthQC.m b/AutomaticQC/imosImpossibleDepthQC.m
@@ -254,8 +254,8 @@
         % read dataset QC parameters if exist and override previous 
         % parameters file
         currentQCtest   = mfilename;
-        zNominalMargin  = readQCparameter(sample_data.toolbox_input_file, currentQCtest, 'zNominalMargin', zNominalMargin);
-        maxAngle        = readQCparameter(sample_data.toolbox_input_file, currentQCtest, 'maxAngle', maxAngle);
+        zNominalMargin  = readDatasetParameter(sample_data.toolbox_input_file, currentQCtest, 'zNominalMargin', zNominalMargin);
+        maxAngle        = readDatasetParameter(sample_data.toolbox_input_file, currentQCtest, 'maxAngle', maxAngle);
 
         paramsLog = ['zNominalMargin=' num2str(zNominalMargin) ', maxAngle=' num2str(maxAngle)];
 
@@ -365,8 +365,8 @@
         end
 
         % write/update dataset QC parameters
-        writeQCparameter(sample_data.toolbox_input_file, currentQCtest, 'zNominalMargin', zNominalMargin);
-        writeQCparameter(sample_data.toolbox_input_file, currentQCtest, 'maxAngle', maxAngle);
+        writeDatasetParameter(sample_data.toolbox_input_file, currentQCtest, 'zNominalMargin', zNominalMargin);
+        writeDatasetParameter(sample_data.toolbox_input_file, currentQCtest, 'maxAngle', maxAngle);
 
         % update climatologyRange info for display
         climatologyRange(p).(['range' paramName]) = [instrumentNominalDepth; instrumentNominalDepth];

diff --git a/AutomaticQC/imosPercentGoodVelocitySetQC.m b/AutomaticQC/imosPercentGoodVelocitySetQC.m
@@ -109,7 +109,7 @@
 % read dataset QC parameters if exist and override previous 
 % parameters file
 currentQCtest = mfilename;
-pgood = readQCparameter(sample_data.toolbox_input_file, currentQCtest, 'pgood', pgood);
+pgood = readDatasetParameter(sample_data.toolbox_input_file, currentQCtest, 'pgood', pgood);
 
 paramsLog = ['pgood=' num2str(pgood)];
 
@@ -148,6 +148,6 @@
 end
 
 % write/update dataset QC parameters
-writeQCparameter(sample_data.toolbox_input_file, currentQCtest, 'pgood', pgood);
+writeDatasetParameter(sample_data.toolbox_input_file, currentQCtest, 'pgood', pgood);
 
 end
diff --git a/AutomaticQC/imosRateOfChangeQC.m b/AutomaticQC/imosRateOfChangeQC.m
@@ -91,7 +91,7 @@
 % read dataset QC parameters if exist and override previous 
 % parameters file
 currentQCtest = mfilename;
-values = readQCparameter(sample_data.toolbox_input_file, currentQCtest, '*', values);
+values = readDatasetParameter(sample_data.toolbox_input_file, currentQCtest, '*', values);
 
 param = strtrim(values{1});
 thresholdExpr = strtrim(values{2});
@@ -225,6 +225,6 @@
 
     % write/update dataset QC parameters
     for i=1:length(param)
-        writeQCparameter(sample_data.toolbox_input_file, currentQCtest, param{i}, thresholdExpr{i});
+        writeDatasetParameter(sample_data.toolbox_input_file, currentQCtest, param{i}, thresholdExpr{i});
     end
 end
diff --git a/AutomaticQC/imosSideLobeVelocitySetQC.m b/AutomaticQC/imosSideLobeVelocitySetQC.m
@@ -119,7 +119,7 @@
 % read dataset QC parameters if exist and override previous 
 % parameters file
 currentQCtest = mfilename;
-nBinSize = readQCparameter(sample_data.toolbox_input_file, currentQCtest, 'nBinSize', nBinSize);
+nBinSize = readDatasetParameter(sample_data.toolbox_input_file, currentQCtest, 'nBinSize', nBinSize);
 
 paramsLog = ['nBinSize=' num2str(nBinSize)];
 
@@ -268,6 +268,6 @@
 end
 
 % write/update dataset QC parameters
-writeQCparameter(sample_data.toolbox_input_file, currentQCtest, 'nBinSize', nBinSize);
+writeDatasetParameter(sample_data.toolbox_input_file, currentQCtest, 'nBinSize', nBinSize);
 
 end
diff --git a/AutomaticQC/imosVerticalSpikeQC.m b/AutomaticQC/imosVerticalSpikeQC.m
@@ -77,7 +77,7 @@
 % read dataset QC parameters if exist and override previous 
 % parameters file
 currentQCtest = mfilename;
-values = readQCparameter(sample_data.toolbox_input_file, currentQCtest, '*', values);
+values = readDatasetParameter(sample_data.toolbox_input_file, currentQCtest, '*', values);
 
 param = strtrim(values{1});
 thresholdExpr = strtrim(values{2});
@@ -197,6 +197,6 @@
 
     % write/update dataset QC parameters
     for i=1:length(param)
-        writeQCparameter(sample_data.toolbox_input_file, currentQCtest, param{i}, thresholdExpr{i});
+        writeDatasetParameter(sample_data.toolbox_input_file, currentQCtest, param{i}, thresholdExpr{i});
     end
 end
diff --git a/AutomaticQC/imosVerticalVelocityQC.m b/AutomaticQC/imosVerticalVelocityQC.m
@@ -94,7 +94,7 @@
 % read dataset QC parameters if exist and override previous 
 % parameters file
 currentQCtest = mfilename;
-vvel = readQCparameter(sample_data.toolbox_input_file, currentQCtest, 'vvel', vvel);
+vvel = readDatasetParameter(sample_data.toolbox_input_file, currentQCtest, 'vvel', vvel);
 
 paramsLog = ['vvel=' num2str(vvel)];
 
@@ -112,6 +112,6 @@
 flags(iPass) = goodFlag;
 
 % write/update dataset QC parameters
-writeQCparameter(sample_data.toolbox_input_file, currentQCtest, 'vvel', vvel);
+writeDatasetParameter(sample_data.toolbox_input_file, currentQCtest, 'vvel', vvel);
 
 end
diff --git a/AutomaticQC/teledyneSetQC.m b/AutomaticQC/teledyneSetQC.m
@@ -145,12 +145,12 @@
 % read dataset QC parameters if exist and override previous 
 % parameters file
 currentQCtest   = mfilename;
-qcthresh.err_vel    = readQCparameter(sample_data.toolbox_input_file, currentQCtest, 'err_vel',     qcthresh.err_vel);
-qcthresh.pgood      = readQCparameter(sample_data.toolbox_input_file, currentQCtest, 'pgood',       qcthresh.pgood);
-qcthresh.cmag       = readQCparameter(sample_data.toolbox_input_file, currentQCtest, 'cmag',        qcthresh.cmag);
-qcthresh.vvel       = readQCparameter(sample_data.toolbox_input_file, currentQCtest, 'vvel',        qcthresh.vvel);
-qcthresh.hvel       = readQCparameter(sample_data.toolbox_input_file, currentQCtest, 'hvel',        qcthresh.hvel);
-qcthresh.ea_thresh  = readQCparameter(sample_data.toolbox_input_file, currentQCtest, 'ea_thresh',   qcthresh.ea_thresh);
+qcthresh.err_vel    = readDatasetParameter(sample_data.toolbox_input_file, currentQCtest, 'err_vel',     qcthresh.err_vel);
+qcthresh.pgood      = readDatasetParameter(sample_data.toolbox_input_file, currentQCtest, 'pgood',       qcthresh.pgood);
+qcthresh.cmag       = readDatasetParameter(sample_data.toolbox_input_file, currentQCtest, 'cmag',        qcthresh.cmag);
+qcthresh.vvel       = readDatasetParameter(sample_data.toolbox_input_file, currentQCtest, 'vvel',        qcthresh.vvel);
+qcthresh.hvel       = readDatasetParameter(sample_data.toolbox_input_file, currentQCtest, 'hvel',        qcthresh.hvel);
+qcthresh.ea_thresh  = readDatasetParameter(sample_data.toolbox_input_file, currentQCtest, 'ea_thresh',   qcthresh.ea_thresh);
 
 paramsLog = ['err_vel=' num2str(qcthresh.err_vel) ', pgood=' num2str(qcthresh.pgood) ...
     ', cmag=' num2str(qcthresh.cmag) ', vvel=' num2str(qcthresh.vvel) ...
@@ -195,12 +195,12 @@
 end
 
 % write/update dataset QC parameters
-writeQCparameter(sample_data.toolbox_input_file, currentQCtest, 'err_vel',  qcthresh.err_vel);
-writeQCparameter(sample_data.toolbox_input_file, currentQCtest, 'pgood',    qcthresh.pgood);
-writeQCparameter(sample_data.toolbox_input_file, currentQCtest, 'cmag',     qcthresh.cmag);
-writeQCparameter(sample_data.toolbox_input_file, currentQCtest, 'vvel',     qcthresh.vvel);
-writeQCparameter(sample_data.toolbox_input_file, currentQCtest, 'hvel',     qcthresh.hvel);
-writeQCparameter(sample_data.toolbox_input_file, currentQCtest, 'ea_thresh',qcthresh.ea_thresh);
+writeDatasetParameter(sample_data.toolbox_input_file, currentQCtest, 'err_vel',  qcthresh.err_vel);
+writeDatasetParameter(sample_data.toolbox_input_file, currentQCtest, 'pgood',    qcthresh.pgood);
+writeDatasetParameter(sample_data.toolbox_input_file, currentQCtest, 'cmag',     qcthresh.cmag);
+writeDatasetParameter(sample_data.toolbox_input_file, currentQCtest, 'vvel',     qcthresh.vvel);
+writeDatasetParameter(sample_data.toolbox_input_file, currentQCtest, 'hvel',     qcthresh.hvel);
+writeDatasetParameter(sample_data.toolbox_input_file, currentQCtest, 'ea_thresh',qcthresh.ea_thresh);
 
 end
 

diff --git a/FlowManager/autoIMOSToolbox.m b/FlowManager/autoIMOSToolbox.m
@@ -191,7 +191,7 @@ function autoIMOSToolbox(toolboxVersion, fieldTrip, dataDir, ppChain, qcChain, e
     if isempty(sample_data), continue; end
 
     raw_data = preprocessManager(sample_data, 'raw', mode, true);
-    qc_data  = preprocessManager(raw_data, 'qc', mode, true);
+    qc_data  = preprocessManager(sample_data, 'qc',  mode, true);
     clear sample_data;
     qc_data  = autoQCManager(qc_data, true);
 

diff --git a/FlowManager/flowManager.m b/FlowManager/flowManager.m
@@ -55,11 +55,11 @@ function flowManager(toolboxVersion)
   for k = 1:length(nonUTCRawData), nonUTCRawData{k}.meta.index = k; end
 
   % preprocess data
-  [rawData, cancel] = preprocessManager(nonUTCRawData, 'raw', mode, false); % only apply TIME to UTC conversion pre-processing routines
+  [autoQCData, cancel] = preprocessManager(nonUTCRawData, 'qc', mode, false);
   if cancel
-      autoQCData = rawData;
+      rawData = nonUTCRawData;
   else
-      autoQCData = preprocessManager(rawData, 'qc',  mode, true);  % apply any pp routine except TIME to UTC conversion, auto is true so that GUI only appears once
+      rawData = preprocessManager(nonUTCRawData, 'raw',  mode, true);  % only apply TIME to UTC conversion pre-processing routines, auto is true so that GUI only appears once
   end
   clear nonUTCRawData;
 

diff --git a/FlowManager/importManager.m b/FlowManager/importManager.m
@@ -85,16 +85,15 @@
   rawFiles    = {};
 
   if ~isempty(ddb) || (~isempty(driver) && ~isempty(connection))
-      [structs rawFiles] = ddbImport(auto, iMooring, ddb, mode);
+      [structs, rawFiles] = ddbImport(auto, iMooring, ddb, mode);
   else
       if auto, error('manual import cannot be automated without deployment database'); end
-      [structs rawFiles] = manualImport(mode);
+      [structs, rawFiles] = manualImport(mode);
   end
 
   % user cancelled
   if isempty(structs), return; end
 
-  dateFmt = readProperty('exportNetCDF.dateFormat');
   qcSet   = str2double(readProperty('toolbox.qc_set'));
   rawFlag = imosQCFlag('raw', qcSet, 'flag');
 
@@ -118,7 +117,7 @@
   end
 end
 
-function [sample_data rawFile]= manualImport(mode)
+function [sample_data, rawFile]= manualImport(mode)
 %MANUALIMPORT Imports a data set by manually prompting the user to select a 
 % raw file, and a parser with which to import it.
 %
@@ -140,7 +139,7 @@
   while true
 
     % prompt the user to select a data file
-    [rawFile path] = uigetfile('*', 'Select Data File', manualDir);
+    [rawFile, path] = uigetfile('*', 'Select Data File', manualDir);
 
     if rawFile == 0, return; end;
 
@@ -158,7 +157,7 @@
 
     % display progress dialog
     progress = waitbar(0,      rawFile, ...
-      'Name',                  'Importing file',...
+      'Name',                  'Importing files',...
       'DefaultTextInterpreter','none');
 
     % import the data
@@ -193,7 +192,7 @@
   end
 end
 
-function [sample_data rawFiles] = ddbImport(auto, iMooring, ddb, mode)
+function [sample_data, rawFiles] = ddbImport(auto, iMooring, ddb, mode)
 %DDBIMPORT Imports data sets using metadata retrieved from a deployment
 % database.
 %
@@ -226,9 +225,9 @@
   while true
       switch mode
           case 'profile'
-              [fieldTrip deps sits dataDir] = getCTDs(auto, isCSV); % one entry is one CTD profile instrument file
+              [fieldTrip, deps, sits, dataDir] = getCTDs(auto, isCSV); % one entry is one CTD profile instrument file
           case 'timeSeries'
-              [fieldTrip deps sits dataDir] = getDeployments(auto, isCSV); % one entry is one moored instrument file
+              [fieldTrip, deps, sits, dataDir] = getDeployments(auto, isCSV); % one entry is one moored instrument file
       end
 
       if isempty(fieldTrip), return; end
@@ -304,21 +303,14 @@
     % find physical files for each deployed instrument
     allFiles = cell(size(deps));
     for k = 1:length(deps)
-
-        switch mode
-            case 'profile'
-                id   = deps(k).FieldTrip;
-            case 'timeSeries'
-                id   = deps(k).DeploymentId;
-        end
 
       rawFile = deps(k).FileName;
 
       hits = fsearch(rawFile, dataDir, 'files');
 
-      % we remove any potential .pqc or .mqc files found (reserved for use
+      % we remove any potential .ppp, .pqc or .mqc files found (reserved for use
       % by the toolbox)
-      reservedExts = {'.pqc', '.mqc'};
+      reservedExts = {'.ppp', '.pqc', '.mqc'};
       for l=1:length(hits)
           [~, ~, ext] = fileparts(hits{l});
           if all(~strcmp(ext, reservedExts))
@@ -327,17 +319,36 @@
       end
     end
 
+    % Sort data_samples
+    %
+    % [B, iX] = sort(A);
+    % =>
+    % A(iX) == B
+    %
+    switch mode
+        case 'timeSeries'
+            % for a mooring, sort instruments by depth
+            % we have to handle the case when InstrumentDepth is not documented
+            instDepths = {deps.InstrumentDepth};
+            instDepths(cellfun(@isempty, instDepths)) = {NaN};
+            instDepths = cell2mat(instDepths);
+
+            [~, iSort] = sort(instDepths);
+            deps = deps(iSort);
+            allFiles = allFiles(iSort);
+    end
+
     % display status dialog to highlight any discrepancies (file not found
     % for a deployment, more than one file found for a deployment)
     if ~auto
-      [deps allFiles] = dataFileStatusDialog(deps, allFiles, isCSV);
+      [deps, allFiles] = dataFileStatusDialog(deps, allFiles, isCSV);
 
       % user cancelled file dialog
       if isempty(deps), continue; end
 
       % display progress dialog
-      progress = waitbar(0,      'Importing file', ...
-        'Name',                  'Importing file',...
+      progress = waitbar(0,      'Importing files', ...
+        'Name',                  'Importing files',...
         'DefaultTextInterpreter','none');
     end