Merge pull request #354 from aodn/2.5.11

2.5.11

DDB/executeCSVQuery.m

@@ -79,11 +79,8 @@
       end
   end
 
-  %get location of csv files:
-  dirnm = readProperty('toolbox.ddb.connection');
-  if isempty(dirnm)
-      dirnm = pwd;
-  end
+  %get location of csv files
+  dirnm = readProperty('toolbox.ddb');
 
   % complete the file name:
   file = fullfile(dirnm, [file '.csv']);

FlowManager/autoIMOSToolbox.m

@@ -158,12 +158,19 @@ function autoIMOSToolbox(toolboxVersion, fieldTrip, dataDir, ppChain, qcChain, e
 [~, sourceFolder] = fileparts(dataDir);
 fprintf('%s\n', ['Processing field trip ' fieldTrip ' from folder ' sourceFolder]);
 
+%check for CSV file import
+isCSV = false;
+ddb = readProperty('toolbox.ddb');
+if isdir(ddb)
+    isCSV = true;
+end
+
 % get infos from current field trip
 switch mode
     case 'profile'
-        [~, deps, sits, dataDir] = getCTDs(true);
+        [~, deps, sits, dataDir] = getCTDs(true, isCSV);
     case 'timeSeries'
-        [~, deps, sits, dataDir] = getDeployments(true);
+        [~, deps, sits, dataDir] = getDeployments(true, isCSV);
 end
 
 if isempty(deps)

FlowManager/importManager.m

@@ -217,12 +217,18 @@
   rawFiles    = {};
   allFiles    = {};
 
+  %check for CSV file import
+  isCSV = false;
+  if isdir(ddb)
+      isCSV = true;
+  end
+
   while true
       switch mode
           case 'profile'
-              [fieldTrip deps sits dataDir] = getCTDs(auto); % one entry is one CTD profile instrument file
+              [fieldTrip deps sits dataDir] = getCTDs(auto, isCSV); % one entry is one CTD profile instrument file
           case 'timeSeries'
-              [fieldTrip deps sits dataDir] = getDeployments(auto); % one entry is one moored instrument file
+              [fieldTrip deps sits dataDir] = getDeployments(auto, isCSV); % one entry is one moored instrument file
       end
 
       if isempty(fieldTrip), return; end
@@ -324,7 +330,7 @@
     % display status dialog to highlight any discrepancies (file not found
     % for a deployment, more than one file found for a deployment)
     if ~auto
-      [deps allFiles] = dataFileStatusDialog(deps, allFiles);
+      [deps allFiles] = dataFileStatusDialog(deps, allFiles, isCSV);
 
       % user cancelled file dialog
       if isempty(deps), continue; end
@@ -362,7 +368,7 @@
           waitbar(k / length(deps), progress, fileDisplay);
       end
       % import data
-      sample_data{end+1} = parse(deps(k), allFiles{k}, parsers, noParserPrompt, mode, ddb);
+      sample_data{end+1} = parse(deps(k), allFiles{k}, parsers, noParserPrompt, mode, isCSV);
       rawFiles{   end+1} = allFiles{k};
 
       if iscell(sample_data{end})
@@ -419,7 +425,7 @@
   % close progress dialog
   if ~auto, close(progress); end
 
-  function sam = parse(deployment, files, parsers, noParserPrompt, mode, ddb)
+  function sam = parse(deployment, files, parsers, noParserPrompt, mode, isCSV)
   %PARSE Parses a raw data file, returns a sample_data struct.
   %
   % Inputs:
@@ -435,7 +441,7 @@
   %   sam        - Struct containing sample data.
 
     % get the appropriate parser function
-    parser = getParserFunc(deployment, parsers, noParserPrompt, ddb);
+    parser = getParserFunc(deployment, parsers, noParserPrompt, isCSV);
     if isnumeric(parser)
       error(['no parser found for instrument ' deployment.InstrumentID]); 
     end
@@ -444,7 +450,7 @@
     sam = parser(files, mode);
   end
 
-  function parser = getParserFunc(deployment, parsers, noParserPrompt, ddb)
+  function parser = getParserFunc(deployment, parsers, noParserPrompt, isCSV)
   %GETPARSERFUNC Searches for a parser function which is able to parse data 
   % for the given deployment.
   %
@@ -459,13 +465,13 @@
   %   parser     - Function handle to the parser function, or 0 if a parser
   %                function wasn't found.
   %
-  if strcmp('csv',ddb)
-      instrument = executeCSVQuery(...
-          'Instruments', 'InstrumentID', deployment.InstrumentID);
-  else
-      instrument = executeDDBQuery(...
-          'Instruments', 'InstrumentID', deployment.InstrumentID);
-  end
+
+    if isCSV
+      executeQueryFunc = @executeCSVQuery;
+    else
+      executeQueryFunc = @executeDDBQuery;
+    end
+    instrument = executeQueryFunc('Instruments', 'InstrumentID', deployment.InstrumentID);
 
     % there should be exactly one instrument 
     if length(instrument) ~= 1
@@ -501,4 +507,4 @@
     % get the parser function handle
     parser = getParser(parser);
   end
-end
+end

GUI/dataFileStatusDialog.m

@@ -1,4 +1,4 @@
-function [deployments files] = dataFileStatusDialog( deployments, files )
+function [deployments files] = dataFileStatusDialog( deployments, files, isCSV )
 %DATAFILESTATUSDIALOG Displays a list of deployments, and raw files for
 % each, allowing the user to verify/change which raw data files map to which
 % deployment.
@@ -16,6 +16,7 @@
 %
 %   files       - Cell array of cell arrays of strings, each containing the
 %                 list of file names corresponding to each deployment.
+%   isCSV       - Logical for whether we use a ddb in .csv format or not.
 %
 % Outputs:
 %   deployments - Same as input, potentially with some deployments removed.
@@ -55,10 +56,11 @@
 % ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
 % POSSIBILITY OF SUCH DAMAGE.
 %
-  narginchk(2,2);
+  narginchk(3,3);
 
   if ~isstruct(deployments), error('deployments must be a struct'); end
   if ~iscell  (files),       error('files must be a cell array');   end
+  if ~islogical(isCSV),      error('isCSV must be a logical');   end
 
   % copy the inputs so we can rollback if the user cancels
   origDeployments = deployments;
@@ -67,7 +69,7 @@
   % get the toolbox execution mode
   mode = readProperty('toolbox.mode');
 
-  deploymentDescs = genDepDescriptions(deployments, files);
+  deploymentDescs = genDepDescriptions(deployments, files, isCSV);
 
   % Sort data_samples
   %
@@ -307,12 +309,10 @@ function fileAddCallback(source,ev)
 
   %% Description generation
 
-  function descs = genDepDescriptions(deployments, files)
+  function descs = genDepDescriptions(deployments, files, isCSV)
   %GENDEPDESCRIPTIONS Creates a cell array of descriptions of the given
   % deployments, suitable for use in the deployments list.
   %
-
-    ddb = readProperty('toolbox.ddb');
 
     % set values for lists
     descs = {};
@@ -336,11 +336,12 @@ function fileAddCallback(source,ev)
 
 
       % get some site information if it exists
-      if strcmp('csv', ddb)
-          site = executeCSVQuery('Sites', 'Site', dep.Site);
+      if isCSV
+          executeQueryFunc = @executeCSVQuery;
       else
-          site = executeDDBQuery('Sites', 'Site', dep.Site);
+          executeQueryFunc = @executeDDBQuery;
       end
+      site = executeQueryFunc('Sites', 'Site', dep.Site);
 
       if ~isempty(site)
 

NetCDF/makeNetCDFCompliant.m

@@ -142,10 +142,16 @@
     sample_data.dimensions{k} = mergeAtts(sample_data.dimensions{k}, dimAtts);
   end
 
+  %check for CSV file import
+  isCSV = false;
+  ddb = readProperty('toolbox.ddb');
+  if isdir(ddb)
+      isCSV = true;
+  end
+
   %
   % variables
   %
-  ddb = readProperty('toolbox.ddb');
   for k = 1:length(sample_data.variables)
 
 
@@ -167,11 +173,11 @@
     if isfield(sample_data.meta, 'deployment')
         iTime = getVar(sample_data.dimensions, 'TIME');
         sample_data.variables{k}.sensor_serial_number = ...
-            getSensorSerialNumber(sample_data.variables{k}.name, sample_data.meta.deployment.InstrumentID, sample_data.dimensions{iTime}.data(1), ddb);
+            getSensorSerialNumber(sample_data.variables{k}.name, sample_data.meta.deployment.InstrumentID, sample_data.dimensions{iTime}.data(1), isCSV);
     elseif isfield(sample_data.meta, 'profile')
         iTime = getVar(sample_data.variables, 'TIME');
         sample_data.variables{k}.sensor_serial_number = ...
-            getSensorSerialNumber(sample_data.variables{k}.name, sample_data.meta.profile.InstrumentID, sample_data.variables{iTime}.data(1), ddb);
+            getSensorSerialNumber(sample_data.variables{k}.name, sample_data.meta.profile.InstrumentID, sample_data.variables{iTime}.data(1), isCSV);
     end
   end
 end
@@ -194,19 +200,20 @@
   end
 end
 
-function target = getSensorSerialNumber ( IMOSParam, InstrumentID, timeFirstSample, ddb )
+function target = getSensorSerialNumber ( IMOSParam, InstrumentID, timeFirstSample, isCSV )
 %GETSENSORSERIALNUMBER gets the sensor serial number associated to an IMOS
 %paramter for a given deployment ID
 %
 
 target = '';
 
 % query the ddb for all sensor config related to this instrument ID
-if strcmp('csv',ddb)
-    InstrumentSensorConfig = executeCSVQuery('InstrumentSensorConfig', 'InstrumentID',   InstrumentID);
+if isCSV
+    executeQueryFunc = @executeCSVQuery;
 else
-    InstrumentSensorConfig = executeDDBQuery('InstrumentSensorConfig', 'InstrumentID',   InstrumentID);
+    executeQueryFunc = @executeDDBQuery;
 end
+InstrumentSensorConfig = executeQueryFunc('InstrumentSensorConfig', 'InstrumentID',   InstrumentID);
 lenConfig = length(InstrumentSensorConfig);
 % only consider relevant config based on timeFirstSample
 for i=1:lenConfig

Parser/convertSBEcnvVar.m

@@ -213,8 +213,15 @@
     % A/D counts to volts (sensor_analog_output 0 to 7)
     case {'v0', 'v1', 'v2', 'v3', 'v4', 'v5', 'v6', 'v7'}
       origName = name;
-      name = ['volt_', getVoltageName(origName, instHeader)];
-      comment = getVoltageComment(origName, procHeader);
+      name = getVoltageName(origName, instHeader);
+      if ~strcmpi(name, 'not_assigned')
+          name = ['volt_', getVoltageName(origName, instHeader)];
+          comment = getVoltageComment(origName, procHeader);
+      else
+          name = '';
+          data = [];
+          comment = '';
+      end
 
     case 'f1'
         if strcmpi(mode, 'profile')
@@ -317,4 +324,6 @@
         end
 end
 
+name = strrep(name, ' ', '_');
+
 end

Parser/readSBE19cnv.m

@@ -80,7 +80,7 @@
             % we deliberately overwrite it assuming the last version is the
             % most relevant
 
-        otherwise
+        case 'timeSeries'
             % if the same parameter appears multiple times, 
             % don't overwrite it in the data struct - append
             % a number to the end of the variable name, as

Preprocessing/timeDriftPP.m

@@ -70,7 +70,12 @@
 % auto logical in input to enable running under batch processing
 if nargin<3, auto=false; end
 
+%check for CSV file import
+isCSV = false;
 ddb = readProperty('toolbox.ddb');
+if isdir(ddb)
+    isCSV = true;
+end
 
 descs           = {};
 nSample         = length(sample_data);
@@ -84,13 +89,10 @@
     descs{k} = genSampleDataDesc(sample_data{k});
 
     %check to see if the offsets are available already from the ddb
-    if strcmp('csv',ddb)
+    if isCSV
         if isfield(sample_data{k}.meta.deployment,'StartOffset')
             startOffsets(k) = sample_data{k}.meta.deployment.StartOffset;
         end
-    end
-
-    if strcmp('csv',ddb)
         if isfield(sample_data{k}.meta.deployment,'EndOffset')
             endOffsets(k) = sample_data{k}.meta.deployment.EndOffset;
         end

Util/getCTDs.m

@@ -1,4 +1,4 @@
-function [fieldTrip ctds sites dataDir] = getCTDs(auto)
+function [fieldTrip ctds sites dataDir] = getCTDs(auto, isCSV)
 %GETCTDS Prompts the user for a field trip ID and data directory.
 % Retrieves and returns the field trip, all ctds from the DDB that
 % are related to the field trip, and the selected data directory.
@@ -7,6 +7,7 @@
 %   auto        - if true, the user is not prompted to select a field
 %                 trip/directory; the values in toolboxProperties are
 %                 used.
+%   isCSV       - If true, look for csv files rather than using database.
 %
 % Outputs:
 %   fieldTrip   - field trip struct - the field trip selected by the user.
@@ -54,7 +55,7 @@
 % prompt the user to select a field trip and
 % directory which contains raw data files
 if ~auto
-    [fieldTrip dataDir] = startDialog('profile');
+    [fieldTrip dataDir] = startDialog('profile', isCSV);
     % if automatic, just get the defaults from toolboxProperties.txt
 else
     dataDir   = readProperty('startDialog.dataDir.profile');
@@ -72,18 +73,23 @@
 fId = fieldTrip.FieldTripID;
 
 % query the ddb for all ctds related to this field trip
-ctds = executeDDBQuery('CTDData', 'FieldTrip', fId);
+if isCSV
+    executeQueryFunc = @executeCSVQuery;
+else
+    executeQueryFunc = @executeDDBQuery;
+end
+ctds = executeQueryFunc('CTDData', 'FieldTrip', fId);
 
 % query the ddb for all sites related to these ctds
 lenDep = length(ctds);
 for i=1:lenDep
     if i==1
-        tempVal = executeDDBQuery('Sites', 'Site', ctds(i).Site);
+        tempVal = executeQueryFunc('Sites', 'Site', ctds(i).Site);
         % A CTDData doesn't necessarily has an associated site, 
         % CTDData already contains some site information
         if ~isempty(tempVal), sites = tempVal; end
     else
-        tempVal = executeDDBQuery('Sites', 'Site', ctds(i).Site);
+        tempVal = executeQueryFunc('Sites', 'Site', ctds(i).Site);
         if ~isempty(tempVal), sites(i) = tempVal; end
     end
 end