diff --git a/+ciapkg/+api/README.md b/+ciapkg/+api/README.md new file mode 100644 index 0000000..66dbb13 --- /dev/null +++ b/+ciapkg/+api/README.md @@ -0,0 +1,7 @@ +# CIAtah API + +Biafra Ahanonu + +All functions in the `ciapkg.api` package are just pass-through functions to the actual underlying functions. This allows users to import all CIAtah functions into their function or script with `import ciapkg.api.*` as opposed to having to do that for each `ciapkg` sub-package. + +Else, users can call nearly all CIAtah functions using `ciapkg.api.[Function Name]`, which allows easier namespacing. \ No newline at end of file diff --git a/+ciapkg/+api/loadMovieList.m b/+ciapkg/+api/loadMovieList.m new file mode 100644 index 0000000..90d98ea --- /dev/null +++ b/+ciapkg/+api/loadMovieList.m @@ -0,0 +1,10 @@ +function [outputMovie, movieDims, nPixels, nFrames] = loadMovieList(movieList, varargin) + % Load movies, automatically detects type (avi, tif, or hdf5) and concatenates if multiple movies in a list. + % NOTE: + % The function assumes input is 2D time series movies with [x y frames] as dimensions + % If movies are different sizes, use largest dimensions and align all movies to top-left corner. + % Biafra Ahanonu + % started: 2013.11.01 + + [outputMovie, movieDims, nPixels, nFrames] = loadMovieList(movieList, 'passArgs', varargin); +end diff --git a/+ciapkg/+api/manageParallelWorkers.m b/+ciapkg/+api/manageParallelWorkers.m new file mode 100644 index 0000000..299878c --- /dev/null +++ b/+ciapkg/+api/manageParallelWorkers.m @@ -0,0 +1,7 @@ +function [success] = manageParallelWorkers(varargin) + % Manages loading and stopping parallel processing workers. + % Biafra Ahanonu + % started: 2015.12.01 + + [success] = manageParallelWorkers('passArgs', varargin); +end \ No newline at end of file diff --git a/+ciapkg/+api/playMovie.m b/+ciapkg/+api/playMovie.m new file mode 100644 index 0000000..c7fa18d --- /dev/null +++ b/+ciapkg/+api/playMovie.m @@ -0,0 +1,3 @@ +function [exitSignal, ostruct] = playMovie(inputMovie, varargin) + [exitSignal, ostruct] = playMovie(inputMovie,'passArgs', varargin); +end \ No newline at end of file diff --git a/+ciapkg/+io/loadDependencies.m b/+ciapkg/+io/loadDependencies.m new file mode 100644 index 0000000..da38ca3 --- /dev/null +++ b/+ciapkg/+io/loadDependencies.m @@ -0,0 +1,119 @@ +function loadDependencies(varargin) + % Download and load CIAtah dependencies. + % Biafra Ahanonu + % started: 2014.07.31 + % 2021.02.01 [15:09:46] branched from CIAtah + % branch from calciumImagingAnalysis 2020.05.07 [15:47:29] + % inputs + % + % outputs + % + + % changelog + % 2020.05.12 [17:40:37] - Updated to enable GUI-less loading of dependencies. In particular for easier unit testing. + % 2020.06.28 [14:25:04] - Added ability for users to force update. + % 2021.01.22 [13:42:36] - NWB from specific release to reduce compatibility errors. + % 2021.02.01 [15:10:41] - Separated into non-class function for use in more functions without needing to load CIAtah class. + % 2021.02.01 [‏‎15:19:40] - Update `_external_programs` to call ciapkg.getDirExternalPrograms() to standardize call across all functions. + % TODO + % Verify all dependencies download and if not ask user to download again. + + %======================== + % DESCRIPTION + options.externalProgramsDir = ciapkg.getDirExternalPrograms(); + options.guiEnabled = 1; + options.dependencyStr = {'downloadMiji','downloadCnmfGithubRepositories','example_downloadTestData','loadMiji','downloadNeuroDataWithoutBorders'}; + + options.dispStr = {'Download Fiji (to run Miji)','Download CNMF, CNMF-E, and CVX code.','Download test one- and two photon datasets.','Load Fiji/Miji into MATLAB path.','Download NWB (NeuroDataWithoutBorders)'}; + % Int vector: index of options.dependencyStr to run by default with no GUI + options.depIdxArray = [1 2 3 5]; + % Binary: 1 = force update even if already downloaded. 0 = skip if already downloaded + options.forceUpdate = 0; + % get options + options = getOptions(options,varargin); + % display(options) + % unpack options into current workspace + % fn=fieldnames(options); + % for i=1:length(fn) + % eval([fn{i} '=options.' fn{i} ';']); + % end + %======================== + + scnsize = get(0,'ScreenSize'); + if ischar(options.dispStr) + options.dispStr = {options.dispStr}; + end + if ischar(options.dependencyStr) + options.dependencyStr = {options.dependencyStr}; + end + dependencyStr = options.dependencyStr; + + dispStr = options.dispStr; + if options.guiEnabled==1 + [depIdxArray, ~] = listdlg('ListString',dispStr,'ListSize',[scnsize(3)*0.3 scnsize(4)*0.3],'Name','Which dependencies to load? (Can select multiple)','InitialValue',options.depIdxArray); + + forceDownloadVec = [0 1]; + [forceUpdate, ~] = listdlg('ListString',{'No - skip installing dependency if already available.','Yes - force update to most recent version of dependency.'},'ListSize',[scnsize(3)*0.3 scnsize(4)*0.3],'Name','Force download/update? (e.g. "Yes" to update dependencies)','InitialValue',[1]); + forceUpdate = forceDownloadVec(forceUpdate); + else + depIdxArray = options.depIdxArray; + forceUpdate = 0; + end + analysisTypeD = dependencyStr(depIdxArray); + dispStr = dispStr(depIdxArray); + for depNo = 1:length(depIdxArray) + disp([10 repmat('>',1,42)]) + disp(dispStr{depNo}) + switch analysisTypeD{depNo} + case 'downloadCnmfGithubRepositories' + [success] = downloadCnmfGithubRepositories('forceUpdate',forceUpdate); + case 'downloadMiji' + depStr = {'Save Fiji to default directory','Save Fiji to custom directory'}; + if options.guiEnabled==1 + [depIdxArray, ~] = listdlg('ListString',depStr,'ListSize',[scnsize(3)*0.2 scnsize(4)*0.25],'Name','Where to save Fiji?'); + else + depIdxArray = 1; + end + depStr = depStr{depIdxArray}; + if depIdxArray==1 + downloadMiji(); + else + downloadMiji('defaultDir',''); + end + % if exist('pathtoMiji','var') + % end + case 'loadMiji' + modelAddOutsideDependencies('miji'); + case 'example_downloadTestData' + example_downloadTestData(); + case 'downloadCellExtraction' + optionsH.forceUpdate = forceUpdate; + optionsH.signalExtractionDir = options.externalProgramsDir; + optionsH.gitNameDisp = {'cellmax_clean','extract'}; + optionsH.gitRepos = {'https://github.com/schnitzer-lab/CELLMax_CLEAN','https://github.com/schnitzer-lab/EXTRACT'}; + optionsH.gitRepos = cellfun(@(x) [x '/archive/master.zip'],optionsH.gitRepos,'UniformOutput',false); + optionsH.outputDir = optionsH.gitNameDisp; + optionsH.gitName = cellfun(@(x) [x '-master'],optionsH.gitNameDisp,'UniformOutput',false); + [success] = downloadGithubRepositories('options',optionsH); + case 'downloadNeuroDataWithoutBorders' + optionsH.forceUpdate = forceUpdate; + optionsH.signalExtractionDir = options.externalProgramsDir; + optionsH.gitNameDisp = {'nwb_schnitzer_lab','yamlmatlab','matnwb'}; + optionsH.gitRepos = {'https://github.com/schnitzer-lab/nwb_schnitzer_lab','https://github.com/ewiger/yamlmatlab'}; + + % 'https://github.com/NeurodataWithoutBorders/matnwb' + optionsH.gitRepos = cellfun(@(x) [x '/archive/master.zip'],optionsH.gitRepos,'UniformOutput',false); + optionsH.gitRepos = [optionsH.gitRepos 'https://github.com/NeurodataWithoutBorders/matnwb/archive/v2.2.5.3.zip']; + optionsH.outputDir = optionsH.gitNameDisp; + optionsH.gitName = cellfun(@(x) [x '-master'],optionsH.gitNameDisp,'UniformOutput',false); + optionsH.gitName{end} = 'matnwb-2.2.5.3'; + [success] = downloadGithubRepositories('options',optionsH); + + % Add NWB folders to path. + ciapkg.nwb.setupNwb; + % obj.loadBatchFunctionFolders; + otherwise + % nothing + end + end +end \ No newline at end of file diff --git a/+ciapkg/+io/updatePkg.m b/+ciapkg/+io/updatePkg.m index 2067843..1a9c6fb 100644 --- a/+ciapkg/+io/updatePkg.m +++ b/+ciapkg/+io/updatePkg.m @@ -20,7 +20,7 @@ 'ciapkg',... 'docs',... 'file_exchange'}; - % Char: GitHub API URL to VERSION file on CIAPKG repository. + % [IGNORE] Char: GitHub API URL to VERSION file on CIAPKG repository. options.versionURL = 'https://api.github.com/repos/bahanonu/calciumImagingAnalysis/contents/ciapkg/VERSION'; % Binary: 1 = pop-up GUI enabled options.showGui = 1; @@ -50,9 +50,9 @@ if verCompare==1 verInfoStr = 'Running most up-to-date version!'; elseif verCompare<1 - verInfoStr = 'Running behind! Initiating update.'; + verInfoStr = 'Running behind! Initiating update [IGNORE for now].'; elseif verCompare>1 - verInfoStr = 'I do not know how, but you are running ahead! [Dev build]'; + verInfoStr = 'I do not know how, but you are running a version ahead! [Dev build]'; end disp(verInfoStr) diff --git a/+ciapkg/+nwb/setupNwb.m b/+ciapkg/+nwb/setupNwb.m index 43f4bd0..11380b5 100644 --- a/+ciapkg/+nwb/setupNwb.m +++ b/+ciapkg/+nwb/setupNwb.m @@ -8,7 +8,7 @@ % % changelog - % + % 2021.02.01 [‏‎15:19:40] - Update `_external_programs` to call ciapkg.getDirExternalPrograms() to standardize call across all functions. % TODO % @@ -16,7 +16,7 @@ % Str: default path for CIAtah options.defaultObjDir = ciapkg.getDir; % Str: default root path where all external programs are stored - options.externalProgramsDir = '_external_programs'; + options.externalProgramsDir = ciapkg.getDirExternalPrograms(); % Str: default path for MatNWB Matlab code options.matnwbDir = 'matnwb'; % get options diff --git a/+ciapkg/getDirExternalPrograms.m b/+ciapkg/getDirExternalPrograms.m new file mode 100644 index 0000000..2debe4d --- /dev/null +++ b/+ciapkg/getDirExternalPrograms.m @@ -0,0 +1,35 @@ +function [externalProgramsDir] = getDirExternalPrograms(varargin) + % Returns the directory where external programs are stored. All functions should call this to find external program directory. + % Biafra Ahanonu + % started: 2021.02.02 [10:55:23] + % inputs + % + % outputs + % + + % changelog + % + % TODO + % + + %======================== + % DESCRIPTION + % options.exampleOption = ''; + % get options + % options = getOptions(options,varargin); + % display(options) + % unpack options into current workspace + % fn=fieldnames(options); + % for i=1:length(fn) + % eval([fn{i} '=options.' fn{i} ';']); + % end + %======================== + + try + externalProgramsDir = [ciapkg.getDir() filesep '_external_programs']; + catch err + disp(repmat('@',1,7)) + disp(getReport(err,'extended','hyperlinks','on')); + disp(repmat('@',1,7)) + end +end \ No newline at end of file diff --git a/+ciapkg/getDirPkg.m b/+ciapkg/getDirPkg.m new file mode 100644 index 0000000..438db51 --- /dev/null +++ b/+ciapkg/getDirPkg.m @@ -0,0 +1,41 @@ +function [ciapkgDir] = getDirPkg(dirType,varargin) + % Standardized location to obtain relevant CIAtah directories, e.g. location of default data folder. + % Biafra Ahanonu + % started: 2020.08.31 [12:46:57] + % inputs + % + % outputs + % + + % changelog + % + % TODO + % + + %======================== + % DESCRIPTION + % options.exampleOption = ''; + % get options + % options = getOptions(options,varargin); + % display(options) + % unpack options into current workspace + % fn=fieldnames(options); + % for i=1:length(fn) + % eval([fn{i} '=options.' fn{i} ';']); + % end + %======================== + + try + switch dirType + case 'data' + ciapkgDir = [ciapkg.getDir() filesep 'data']; + otherwise + ciapkgDir = ''; + disp('Incorrect input, returning null.') + end + catch err + disp(repmat('@',1,7)) + disp(getReport(err,'extended','hyperlinks','on')); + disp(repmat('@',1,7)) + end +end \ No newline at end of file diff --git a/+ciapkg/version.m b/+ciapkg/version.m index 024ca60..22d033d 100644 --- a/+ciapkg/version.m +++ b/+ciapkg/version.m @@ -34,7 +34,7 @@ % versionStr = verStr{1}; % dateTimeStr = num2str(verStr{2}); - verStr = readtable([ciapkg.getDir filesep 'ciapkg' filesep 'VERSION'],'ReadVariableNames',0,'FileType','text','Format','auto','TextType','string') + verStr = readtable([ciapkg.getDir filesep 'ciapkg' filesep 'VERSION'],'ReadVariableNames',0,'FileType','text','Format','auto','TextType','string'); versionStr = verStr{1,1}{1}; dateTimeStr = num2str(verStr{2,1}{1}); catch diff --git a/+ciapkg/versionOnline.m b/+ciapkg/versionOnline.m index 47a8976..3ebdd78 100644 --- a/+ciapkg/versionOnline.m +++ b/+ciapkg/versionOnline.m @@ -1,4 +1,4 @@ -function [onlineVersion] = versionOnline(varargin) +function [onlineVersion, dateTimeStr] = versionOnline(varargin) % Obtains the online repository version. % Biafra Ahanonu % started: 2020.08.18 [‏‎11:16:56] @@ -8,7 +8,7 @@ % % changelog - % + % 2021.02.02 [13:42:19] - Updated to handle new VERSION file that includes datestamp on 2nd line. % TODO % @@ -29,6 +29,8 @@ try success = 0; + onlineVersion = ''; + dateTimeStr = ''; % Get version information online % Get information about specific version file online using GitHub API @@ -40,10 +42,17 @@ disp('Could not dowload CIAPKG version information.') return; end + if ~isempty(regexp(onlineVersion,'\n')) + onlineVersionTmp = strsplit(onlineVersion,'\n'); + onlineVersion = onlineVersionTmp{1}; + dateTimeStr = onlineVersionTmp{2}; + end end success = 1; catch err onlineVersion = ''; + dateTimeStr = ''; + success = 0; disp(repmat('@',1,7)) disp(getReport(err,'extended','hyperlinks','on')); disp(repmat('@',1,7)) diff --git a/.gitignore b/.gitignore index b26f6fe..cbd099b 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,10 @@ _external_programs/*/ _external_programs/cnmf_current _external_programs/cnmfe _external_programs/cvx_rd +_external_programs/matnwb +_external_programs/nwb_schnitzer_lab +_external_programs/yamlmatlab +_external_programs/normcorre # Signal extraction dependency zips _external_programs/*.zip diff --git a/@calciumImagingAnalysis/loadDependencies.m b/@calciumImagingAnalysis/loadDependencies.m index abfa6aa..af09f91 100644 --- a/@calciumImagingAnalysis/loadDependencies.m +++ b/@calciumImagingAnalysis/loadDependencies.m @@ -11,6 +11,7 @@ % 2020.05.12 [17:40:37] - Updated to enable GUI-less loading of dependencies. In particular for easier unit testing. % 2020.06.28 [14:25:04] - Added ability for users to force update. % 2021.01.22 [13:42:36] - NWB from specific release to reduce compatibility errors. + % 2021.02.01 [15:10:41] - Calls non-class function for use in more functions without needing to load CIAtah class. % TODO % Verify all dependencies download and if not ask user to download again. @@ -32,74 +33,83 @@ % end %======================== - scnsize = get(0,'ScreenSize'); - dependencyStr = options.dependencyStr; - dispStr = options.dispStr; - if obj.guiEnabled==1 - [depIdxArray, ~] = listdlg('ListString',dispStr,'ListSize',[scnsize(3)*0.3 scnsize(4)*0.3],'Name','Which dependencies to load? (Can select multiple)','InitialValue',options.depIdxArray); + % scnsize = get(0,'ScreenSize'); + % dependencyStr = options.dependencyStr; + % dispStr = options.dispStr; + % if obj.guiEnabled==1 + % [depIdxArray, ~] = listdlg('ListString',dispStr,'ListSize',[scnsize(3)*0.3 scnsize(4)*0.3],'Name','Which dependencies to load? (Can select multiple)','InitialValue',options.depIdxArray); - forceDownloadVec = [0 1]; - [forceUpdate, ~] = listdlg('ListString',{'No - skip installing dependency if already available.','Yes - force update to most recent version of dependency.'},'ListSize',[scnsize(3)*0.3 scnsize(4)*0.3],'Name','Force download/update? (e.g. "Yes" to update dependencies)','InitialValue',[1]); - forceUpdate = forceDownloadVec(forceUpdate); - else - depIdxArray = options.depIdxArray; - forceUpdate = 0; - end - analysisTypeD = dependencyStr(depIdxArray); - dispStr = dispStr(depIdxArray); - for depNo = 1:length(depIdxArray) - disp([10 repmat('>',1,42)]) - disp(dispStr{depNo}) - switch analysisTypeD{depNo} - case 'downloadCnmfGithubRepositories' - [success] = downloadCnmfGithubRepositories('forceUpdate',forceUpdate); - case 'downloadMiji' - depStr = {'Save Fiji to default directory','Save Fiji to custom directory'}; - if obj.guiEnabled==1 - [depIdxArray, ~] = listdlg('ListString',depStr,'ListSize',[scnsize(3)*0.2 scnsize(4)*0.25],'Name','Where to save Fiji?'); - else - depIdxArray = 1; - end - depStr = depStr{depIdxArray}; - if depIdxArray==1 - downloadMiji(); - else - downloadMiji('defaultDir',''); - end - % if exist('pathtoMiji','var') - % end - case 'loadMiji' - modelAddOutsideDependencies('miji'); - case 'example_downloadTestData' - example_downloadTestData(); - case 'downloadCellExtraction' - optionsH.forceUpdate = forceUpdate; - optionsH.signalExtractionDir = obj.externalProgramsDir; - optionsH.gitNameDisp = {'cellmax_clean','extract'}; - optionsH.gitRepos = {'https://github.com/schnitzer-lab/CELLMax_CLEAN','https://github.com/schnitzer-lab/EXTRACT'}; - optionsH.gitRepos = cellfun(@(x) [x '/archive/master.zip'],optionsH.gitRepos,'UniformOutput',false); - optionsH.outputDir = optionsH.gitNameDisp; - optionsH.gitName = cellfun(@(x) [x '-master'],optionsH.gitNameDisp,'UniformOutput',false); - [success] = downloadGithubRepositories('options',optionsH); - case 'downloadNeuroDataWithoutBorders' - optionsH.forceUpdate = forceUpdate; - optionsH.signalExtractionDir = obj.externalProgramsDir; - optionsH.gitNameDisp = {'nwb_schnitzer_lab','yamlmatlab','matnwb'}; - optionsH.gitRepos = {'https://github.com/schnitzer-lab/nwb_schnitzer_lab','https://github.com/ewiger/yamlmatlab'}; + % forceDownloadVec = [0 1]; + % [forceUpdate, ~] = listdlg('ListString',{'No - skip installing dependency if already available.','Yes - force update to most recent version of dependency.'},'ListSize',[scnsize(3)*0.3 scnsize(4)*0.3],'Name','Force download/update? (e.g. "Yes" to update dependencies)','InitialValue',[1]); + % forceUpdate = forceDownloadVec(forceUpdate); + % else + % depIdxArray = options.depIdxArray; + % forceUpdate = 0; + % % sopts.forceUpdate = 1; + % end + + sopts.guiEnabled = obj.guiEnabled; + sopts.dependencyStr = options.dependencyStr; + sopts.dispStr = options.dispStr; + sopts.depIdxArray = options.depIdxArray; + + ciapkg.io.loadDependencies('options',sopts); - % 'https://github.com/NeurodataWithoutBorders/matnwb' - optionsH.gitRepos = cellfun(@(x) [x '/archive/master.zip'],optionsH.gitRepos,'UniformOutput',false); - optionsH.gitRepos = [optionsH.gitRepos 'https://github.com/NeurodataWithoutBorders/matnwb/archive/v2.2.5.3.zip']; - optionsH.outputDir = optionsH.gitNameDisp; - optionsH.gitName = cellfun(@(x) [x '-master'],optionsH.gitNameDisp,'UniformOutput',false); - optionsH.gitName{end} = 'matnwb-2.2.5.3'; - [success] = downloadGithubRepositories('options',optionsH); + % analysisTypeD = dependencyStr(depIdxArray); + % dispStr = dispStr(depIdxArray); + % for depNo = 1:length(depIdxArray) + % disp([10 repmat('>',1,42)]) + % disp(dispStr{depNo}) + % switch analysisTypeD{depNo} + % case 'downloadCnmfGithubRepositories' + % [success] = downloadCnmfGithubRepositories('forceUpdate',forceUpdate); + % case 'downloadMiji' + % depStr = {'Save Fiji to default directory','Save Fiji to custom directory'}; + % if obj.guiEnabled==1 + % [depIdxArray, ~] = listdlg('ListString',depStr,'ListSize',[scnsize(3)*0.2 scnsize(4)*0.25],'Name','Where to save Fiji?'); + % else + % depIdxArray = 1; + % end + % depStr = depStr{depIdxArray}; + % if depIdxArray==1 + % downloadMiji(); + % else + % downloadMiji('defaultDir',''); + % end + % % if exist('pathtoMiji','var') + % % end + % case 'loadMiji' + % modelAddOutsideDependencies('miji'); + % case 'example_downloadTestData' + % example_downloadTestData(); + % case 'downloadCellExtraction' + % optionsH.forceUpdate = forceUpdate; + % optionsH.signalExtractionDir = obj.externalProgramsDir; + % optionsH.gitNameDisp = {'cellmax_clean','extract'}; + % optionsH.gitRepos = {'https://github.com/schnitzer-lab/CELLMax_CLEAN','https://github.com/schnitzer-lab/EXTRACT'}; + % optionsH.gitRepos = cellfun(@(x) [x '/archive/master.zip'],optionsH.gitRepos,'UniformOutput',false); + % optionsH.outputDir = optionsH.gitNameDisp; + % optionsH.gitName = cellfun(@(x) [x '-master'],optionsH.gitNameDisp,'UniformOutput',false); + % [success] = downloadGithubRepositories('options',optionsH); + % case 'downloadNeuroDataWithoutBorders' + % optionsH.forceUpdate = forceUpdate; + % optionsH.signalExtractionDir = obj.externalProgramsDir; + % optionsH.gitNameDisp = {'nwb_schnitzer_lab','yamlmatlab','matnwb'}; + % optionsH.gitRepos = {'https://github.com/schnitzer-lab/nwb_schnitzer_lab','https://github.com/ewiger/yamlmatlab'}; - % Add NWB folders to path. - ciapkg.nwb.setupNwb; - % obj.loadBatchFunctionFolders; - otherwise - % nothing - end - end + % % 'https://github.com/NeurodataWithoutBorders/matnwb' + % optionsH.gitRepos = cellfun(@(x) [x '/archive/master.zip'],optionsH.gitRepos,'UniformOutput',false); + % optionsH.gitRepos = [optionsH.gitRepos 'https://github.com/NeurodataWithoutBorders/matnwb/archive/v2.2.5.3.zip']; + % optionsH.outputDir = optionsH.gitNameDisp; + % optionsH.gitName = cellfun(@(x) [x '-master'],optionsH.gitNameDisp,'UniformOutput',false); + % optionsH.gitName{end} = 'matnwb-2.2.5.3'; + % [success] = downloadGithubRepositories('options',optionsH); + + % % Add NWB folders to path. + % ciapkg.nwb.setupNwb; + % % obj.loadBatchFunctionFolders; + % otherwise + % % nothing + % end + % end end \ No newline at end of file diff --git a/@calciumImagingAnalysis/modelAddNewFolders.m b/@calciumImagingAnalysis/modelAddNewFolders.m index 87c2cf3..6ca816e 100644 --- a/@calciumImagingAnalysis/modelAddNewFolders.m +++ b/@calciumImagingAnalysis/modelAddNewFolders.m @@ -7,9 +7,13 @@ % 2019.11.18 [15:15:47] - Add the ability for users to use GUI to add folders as alternative option. % 2020.01.16 [12:28:29] - Choosing how to enter files and manual enter list of files now uses uicontrol and figure to reduce number of pop-ups and increase flexibility. % 2020.05.07 [17:22:20] - Adding option to quickly add all the example downloaded folders. + % 2021.01.24 [14:03:44] - Added support for direct input of method type, useful for command-line or unit testing. + % 2021.02.02 [11:27:21] - 'Add CIAtah example folders.' now adds the absolute path to avoid path errors if user changes Matlab current directory. %======================== % Cell array of folders to add, particularly for GUI-less operations options.folderCellArray = {}; + % Str: + options.inputMethod = ''; % get options options = getOptions(options,varargin); % display(options) @@ -23,39 +27,44 @@ nExistingFolders = length(obj.inputFolders); if isempty(options.folderCellArray) sel = 0 - usrIdxChoiceStr = {... - 'manually enter folders to list',... - 'GUI select folders',... - 'Add calciumImagingAnalysis example folders.'}; - scnsize = get(0,'ScreenSize'); - try - hFig = figure; - uicontrol('Style','Text','String',['How to add folders to calciumImagingAnalysis?'],'Units','normalized','Position',[5 89 90 10]/100,'BackgroundColor','white','HorizontalAlignment','Left','FontWeight','bold'); - hListbox = uicontrol(hFig, 'style','listbox','Units', 'normalized','position',[5,5,90,80]/100, 'string',usrIdxChoiceStr,'Value',1); - set(hListbox,'Max',2,'Min',0); - set(hListbox,'KeyPressFcn',@(src,evnt)onKeyPressRelease(evnt,'press',hFig)) - figure(hFig) - uicontrol(hListbox) - set(hFig, 'KeyPressFcn', @(source,eventdata) figure(hFig)); - uiwait(hFig) - catch err - disp(repmat('@',1,7)) - disp(getReport(err,'extended','hyperlinks','on')); - disp(repmat('@',1,7)) + if isempty(options.inputMethod) + usrIdxChoiceStr = {... + 'manually enter folders to list',... + 'GUI select folders',... + 'Add CIAtah example folders.'}; + scnsize = get(0,'ScreenSize'); + try + hFig = figure; + uicontrol('Style','Text','String',['How to add folders to CIAtah?'],'Units','normalized','Position',[5 89 90 10]/100,'BackgroundColor','white','HorizontalAlignment','Left','FontWeight','bold'); + hListbox = uicontrol(hFig, 'style','listbox','Units', 'normalized','position',[5,5,90,80]/100, 'string',usrIdxChoiceStr,'Value',1); + set(hListbox,'Max',2,'Min',0); + set(hListbox,'KeyPressFcn',@(src,evnt)onKeyPressRelease(evnt,'press',hFig)) + figure(hFig) + uicontrol(hListbox) + set(hFig, 'KeyPressFcn', @(source,eventdata) figure(hFig)); + uiwait(hFig) + catch err + disp(repmat('@',1,7)) + disp(getReport(err,'extended','hyperlinks','on')); + disp(repmat('@',1,7)) - [sel, ok] = listdlg('ListString',usrIdxChoiceStr,'ListSize',[scnsize(3)*0.3 scnsize(4)*0.3],'Name','How to add folders to calciumImagingAnalysis?'); + [sel, ok] = listdlg('ListString',usrIdxChoiceStr,'ListSize',[scnsize(3)*0.3 scnsize(4)*0.3],'Name','How to add folders to CIAtah?'); + end + inputMethod = usrIdxChoiceStr{sel}; + else + inputMethod = options.inputMethod; end - inputMethod = usrIdxChoiceStr{sel}; switch inputMethod - case 'Add calciumImagingAnalysis example folders.' + case 'Add CIAtah example folders.' disp('Adding example folders to path...') + dataDir = ciapkg.getDirPkg('data'); newFolderList = {... - ['data' filesep '2014_04_01_p203_m19_check01'],... - ['data' filesep 'batch' filesep '2014_08_05_p104_m19_PAV08'],... - ['data' filesep 'batch' filesep '2014_08_06_p104_m19_PAV09'],... - ['data' filesep 'batch' filesep '2014_08_07_p104_m19_PAV10'],... - ['data' filesep 'twoPhoton' filesep '2017_04_16_p485_m487_runningWheel02']... + [dataDir filesep '2014_04_01_p203_m19_check01'],... + [dataDir filesep 'batch' filesep '2014_08_05_p104_m19_PAV08'],... + [dataDir filesep 'batch' filesep '2014_08_06_p104_m19_PAV09'],... + [dataDir filesep 'batch' filesep '2014_08_07_p104_m19_PAV10'],... + [dataDir filesep 'twoPhoton' filesep '2017_04_16_p485_m487_runningWheel02']... }; disp(newFolderList) nNewFolders = length(newFolderList); @@ -80,7 +89,7 @@ figure(hFig) - uicontrol('Style','Text','String',['Adding folders to calciumImagingAnalysis object.'],'Units','normalized','Position',[5 95 90 3]/100,'BackgroundColor','white','HorizontalAlignment','Left','FontWeight','bold'); + uicontrol('Style','Text','String',['Adding folders to CIAtah object.'],'Units','normalized','Position',[5 95 90 3]/100,'BackgroundColor','white','HorizontalAlignment','Left','FontWeight','bold'); uicontrol('Style','Text','String',['One new line per folder path. Enter folder path WITHOUT any single/double quotation marks around the path.'],'Units','normalized','Position',[5 90 90 6]/100,'BackgroundColor','white','HorizontalAlignment','Left'); exitHandle = uicontrol('style','pushbutton','Units', 'normalized','position',[5 85 50 3]/100,'FontSize',9,'string','Click here to finish','callback',@subfxnCloseFig,'HorizontalAlignment','Left'); @@ -97,7 +106,7 @@ AddOpts.WindowStyle='normal'; AddOpts.Interpreter='tex'; % inputdlg - newFolderList = inputdlgcol('One new line per folder path. Enter folder path WITHOUT any single/double quotation marks around the path.','Adding folders to calciumImagingAnalysis object.',[21 150],{''},AddOpts); + newFolderList = inputdlgcol('One new line per folder path. Enter folder path WITHOUT any single/double quotation marks around the path.','Adding folders to CIAtah object.',[21 150],{''},AddOpts); if isempty(newFolderList) warning('No folders given. Please re-run modelAddNewFolders.') return diff --git a/@calciumImagingAnalysis/modelExtractSignalsFromMovie.m b/@calciumImagingAnalysis/modelExtractSignalsFromMovie.m index a802939..104788e 100644 --- a/@calciumImagingAnalysis/modelExtractSignalsFromMovie.m +++ b/@calciumImagingAnalysis/modelExtractSignalsFromMovie.m @@ -20,12 +20,13 @@ % 2019.10.29 [17:21:23] - Added a check to make sure that filenames produced are valid MATLAB ones for settings, e.g. for CNMF-e. % 2019.11.10 [20:34:42] - Add a warning with some common tips for users if error during cell extraction. Skip modelVarsFromFiles and viewObjmaps loading to reduce user confusion for any folders that had issues during cell extraction. % 2020.05.08 [20:01:52] - Make creation of settings an explicit option that the user can change. + % 2021.02.01 [‏‎15:19:40] - Update `_external_programs` to call ciapkg.getDirExternalPrograms() to standardize call across all functions. % TODO % %======================== % Root path for external signal-extraction algorithm folders. - options.signalExtractionRootPath = [ciapkg.getDir() filesep '_external_programs']; + options.signalExtractionRootPath = ciapkg.getDirExternalPrograms(); % 1 = save NWB output, 2 = do not save NWB output. options.saveNwbOutput = 0; % Str: save to this sub-folder of analyzed folder, leave blank to save in root folder. @@ -463,7 +464,8 @@ function subfxnSaveNwbFiles(inputImages,inputTraces) function getAlgorithmRootPath(algorithmFile,algorithmName,obj,rootFlag) % First try to automatically add the folder try - foundFiles = dir(fullfile([obj.defaultObjDir filesep obj.externalProgramsDir], ['**\' algorithmFile ''])); + % foundFiles = dir(fullfile([obj.defaultObjDir filesep obj.externalProgramsDir], ['**\' algorithmFile ''])); + foundFiles = dir(fullfile([obj.externalProgramsDir], ['**\' algorithmFile ''])); pathToAdd = foundFiles.folder; if rootFlag==1 [pathToAdd,~,~] = fileparts(pathToAdd); @@ -1146,6 +1148,347 @@ function runPCAICASignalFinder() % Save output in NWB format if requested by user. subfxnSaveNwbFiles(IcaFilters,{IcaTraces}); end + function [emOptions] = runCELLMaxSignalFinder() % runEMSignalFinder() + % emOptions.dsMovieDatasetName = options.datasetName; + % emOptions.movieDatasetName = options.datasetName; + loadBatchFxns('loadEverything'); + movieList = getFileList(obj.inputFolders{obj.fileNum}, fileFilterRegexp); + + % The second upsampled movie if there is one + movieListAlt = getFileList(obj.inputFolders{obj.fileNum}, fileFilterRegexpAltCellExtraction); + + % movieFilename=[]; + % upsampledMovieList = getFileList(thisDir, fileFilterRegexp); + % mpiprofile on + % emOptions.CELLMaxoptions = emOptions.EMoptions; + display(['input movie: ' movieList{1}]) + + % ===================== + clear emOptions; + + if strcmp(options.CELLMax.initMethod,'grid') + emOptions.CELLMaxoptions.initMethod = 'grid'; + elseif strcmp(options.CELLMax.initMethod,'ica') + emOptions.CELLMaxoptions.initMethod='ica'; + end + emOptions.CELLMaxoptions.gridSpacing = gridSpacing.(obj.subjectStr{obj.fileNum}); + emOptions.CELLMaxoptions.gridWidth = gridWidth.(obj.subjectStr{obj.fileNum}); + if ~isempty(options.CELLMax.gridSpacing) + emOptions.CELLMaxoptions.gridSpacing = options.CELLMax.gridSpacing; + emOptions.CELLMaxoptions.gridWidth = options.CELLMax.gridWidth; + end + emOptions.useParallel = options.useParallel; + emOptions.CELLMaxoptions.inputSizeManual = 0; + + emOptions.CELLMaxoptions.subsampleMethod = options.CELLMax.subsampleMethod; + + % [maxIters nMovieFrames] + % options.subsampleFrameMatrix = []; + % [1 nMovieFrames] - vector of frames to use in a movie + % options.subsampleFrameVector = []; + % options.selectRandomFrames=1; + % options.numFramesRandom=2000; + % 0 to 1, percentage of frames per iteration to select + emOptions.CELLMaxoptions.percentFramesPerIteration = options.CELLMax.percentFramesPerIteration; + % subsampleMethod = 'resampleRemaining', fr + emOptions.CELLMaxoptions.percentRemainingSubsample = 1; + emOptions.CELLMaxoptions.maxSqSize = options.CELLMax.maxSqSize; + emOptions.CELLMaxoptions.sqOverlap = options.CELLMax.sqOverlap; + emOptions.CELLMaxoptions.percentFramesPCAICA = options.CELLMax.percentFramesPCAICA; + emOptions.CELLMaxoptions.useGPU = options.CELLMax.useGPU; + + emOptions.CELLMaxoptions.sizeThresh = options.CELLMax.sizeThresh; + emOptions.CELLMaxoptions.sizeThreshMax = options.CELLMax.sizeThreshMax; + emOptions.CELLMaxoptions.areaOverlapThresh = options.CELLMax.areaOverlapThresh; + emOptions.CELLMaxoptions.removeCorrProbs = options.CELLMax.removeCorrProbs; + emOptions.CELLMaxoptions.distanceThresh = options.CELLMax.distanceThresh; + emOptions.CELLMaxoptions.corrRemovalAreaOverlapThresh = options.CELLMax.corrRemovalAreaOverlapThresh; + emOptions.CELLMaxoptions.threshForElim = options.CELLMax.threshForElim; + emOptions.CELLMaxoptions.scaledPhiCorrThresh = options.CELLMax.scaledPhiCorrThresh; + emOptions.CELLMaxoptions.runMovieImageCorrThreshold = options.CELLMax.runMovieImageCorrThreshold; + emOptions.CELLMaxoptions.movieImageCorrThreshold = options.CELLMax.movieImageCorrThreshold; + emOptions.CELLMaxoptions.removeAutoCorrThres = options.CELLMax.removeAutoCorrThres; + + emOptions.CELLMaxoptions.loadPreviousChunks = options.CELLMax.loadPreviousChunks; + + emOptions.CELLMaxoptions.numSigmasThresh = options.CELLMax.numSigmasThresh; + emOptions.CELLMaxoptions.numPhotonsPerSigma = options.CELLMax.numPhotonsPerSigma; + + emOptions.CELLMaxoptions.downsampleFactorTime = options.CELLMax.downsampleFactorTime; + emOptions.CELLMaxoptions.downsampleFactorSpace = options.CELLMax.downsampleFactorSpace; + emOptions.CELLMaxoptions.dsInitializeThreshold = options.CELLMax.dsInitializeThreshold; + emOptions.CELLMaxoptions.upsampleFullIters = options.CELLMax.upsampleFullIters; + + emOptions.CELLMaxoptions.spatialFilterMovie = options.CELLMax.spatialFilterMovie; + + emOptions.CELLMaxoptions.useSparseImageMatrix = options.CELLMax.useSparseImageMatrix; + emOptions.CELLMaxoptions.exitEarlySaveSparse = options.CELLMax.exitEarlySaveSparse; + emOptions.CELLMaxoptions.numFramesSampleFitNoiseSigma = options.CELLMax.numFramesSampleFitNoiseSigma; + emOptions.CELLMaxoptions.recalculateFinalTraces = options.CELLMax.recalculateFinalTraces; + + if options.defaultOptions==0 + emOptions.CELLMaxoptions.localICimgs = []; + emOptions.CELLMaxoptions.localICtraces = []; + emOptions.CELLMaxoptions.minIters = options.CELLMax.minIters; + emOptions.CELLMaxoptions.maxIters = options.CELLMax.minIters; + emOptions.CELLMaxoptions.inputSizeManual = 0; + emOptions.CELLMaxoptions.numSigmasThresh = 0.5; + emOptions.CELLMaxoptions.nParallelWorkers = options.numWorkers; + emOptions.CELLMaxoptions.generateNovelSeed = 1; + % emOptions.CELLMaxoptions.randNumGenSeed = 2; + movieDims = loadMovieList(movieList{1},'getMovieDims',1,'inputDatasetName',obj.inputDatasetName); + emOptions.CELLMaxoptions.numFramesRandom = round(movieDims.z*options.CELLMax.percentFramesPerIteration); + if emOptions.CELLMaxoptions.numFramesRandom<3e3 + emOptions.CELLMaxoptions.numFramesRandom = 3e3; + end + emOptions.CELLMaxoptions.readMovieChunks = options.CELLMax.readMovieChunks; + end + + emOptions.movieDatasetName = obj.inputDatasetName; + emOptions.CELLMaxoptions.movieFilename = movieList{1}; + if isempty(movieListAlt) + emOptions.movieFilenameAlt = ''; + else + emOptions.movieFilenameAlt = movieListAlt{1}; + end + % ===================== + + fn_structdisp(emOptions); + + if options.profiler==1 + currentDateTimeStr = datestr(now,'yyyymmdd_HHMM','local'); + profilerSaveLocation = [obj.inputFolders{obj.fileNum} filesep 'profilerCELLMax_' currentDateTimeStr]; + display(['Profiler will be saved to: ' profilerSaveLocation]) + profile on + end + + % [emAnalysisOutput, ~] = CELLMax_Wrapper(movieList{1},'options',emOptions); + [emAnalysisOutput, ~] = cellmax.runCELLMax(movieList{1},'options',emOptions); + + if options.profiler==1 + profile off + profsave(profile('info'),profilerSaveLocation); + end + % [emAnalysisOutput, ~] = EM_CellFind_Wrapper(movieList{1},[],'options',emOptions); + % emOptions.CELLMaxoptions.sqSizeX = NaN; + % emOptions.CELLMaxoptions.sqSizeY = NaN; + + emOptions.CELLMaxoptions.sqSizeX = []; + emOptions.CELLMaxoptions.sqSizeY = []; + % emAnalysisOutput.dsCellTraces = emAnalysisOutput.cellTraces; + % emOptions.CELLMaxoptions.numSignalsDetected = size(emAnalysisOutput.dsCellTraces,1); + emOptions.CELLMaxoptions.numSignalsDetected = size(emAnalysisOutput.cellTraces,1); + emOptions.versionCellmax = emAnalysisOutput.versionCellmax; + % emOptions.EMoptions = emOptions.CELLMaxoptions; + % mpiprofile off + % mpiprofile viewer + % pause + + % output.cellImages : images representing sources found (candidate cells). not all will be cells. Size is [x y numCells] + % output.centroids : centroids of each cell image, x (horizontal) and then y (vertical). Size is [numCells 2] + % output.convexHulls : convex hull (line tracing around edge) of each cell, in x then y. Cell Array, Size is [numCells 1], each entry is hull of one cell. + % output.dsEventTimes : event timings on the down sampled probability traces. + % output.dsScaledProbabilities : a scaled probability trace for each cell, from the downsampled movie. Can be used as a denoised fluorescence trace. + % output.dsCellTraces : fluorescence traces for each cell, from the temporally downsampled movie. Size is [numCells numFrames] for numFrames of downsampled movie + % output.cellTraces : fluorescence traces for each cell, from the full temporal resolution movie. Size is [numCells numFrames] for numFrames of full movie + % output.eventTimes : event timings as output by detectEvents. + % output.EMoptions : options that EM was run with. Good to keep for recordkeeping purposes. + + emOptions.time.startTime = startTime; + emOptions.time.endTime = toc(startTime); + emOptions.time.cellmaxRuntime = emAnalysisOutput.runtime; + try + emOptions.time.cellmaxRuntime = emAnalysisOutput.runtimeWithIO; + catch + end + emAnalysisOutput + + if strcmp(signalExtractionMethod{signalExtractNo},'CELLMax') + % Save CELLMax output using the cellmax output structure name, e.g. cellmaxAnalysisOutput + structSaveName = obj.extractionMethodStructVarname.(obj.signalExtractionMethod); + tmpStruct.(structSaveName) = emAnalysisOutput; + tmpStruct.emOptions = emOptions; + % ======= + % save output components + for i=1:length(saveID) + savestring = [thisDirSaveStr saveID{i}]; + display(['saving: ' savestring]) + save(savestring,'-struct', 'tmpStruct','-v7.3'); + % save(savestring,saveVariable{i},'emOptions'); + end + % ======= + else + % ======= + % save output components + for i=1:length(saveID) + savestring = [thisDirSaveStr saveID{i}]; + display(['saving: ' savestring]) + save(savestring,saveVariable{i},'-v7.3','emOptions'); + % save(savestring,saveVariable{i},'emOptions'); + end + % ======= + end + + % Save output in NWB format if requested by user. + subfxnSaveNwbFiles(emAnalysisOutput.cellImages,{emAnalysisOutput.scaledProbability,emAnalysisOutput.cellTraces}); + + loadBatchFxns(); + end + function [extractAnalysisOutput] = runEXTRACTSignalFinder() + movieList = getFileList(obj.inputFolders{obj.fileNum}, fileFilterRegexp); + [inputMovie thisMovieSize Npixels Ntime] = loadMovieList(movieList,'convertToDouble',0,'inputDatasetName',obj.inputDatasetName,'treatMoviesAsContinuous',1); + inputMovie(isnan(inputMovie)) = 0; + + % opts.movie_dataset = obj.inputDatasetName; + % % opts.save_to_movie_dir = 1; + % % % make larger if using 2x downsampled movie + % % opts.spat_linfilt_halfwidth = 2; + % % opts.ss_cell_size_threshold = 5; + % % opts.spat_medfilt_enabled = 0; + % % opts.trim_pixels = 0.4; + % % opts.verbos = 2; + % % opts.disableGPU = 1; + + % % options.turboreg = getFxnSettings(); + % % options.turboreg + % % options.datasetName = options.turboreg.datasetName; + + % % settingDefaults = struct(... + % % 'movie_dataset',{{'/1','/Movie','/movie'}},... + % % 'save_to_movie_dir', {{1,0}},... + % % 'spat_linfilt_halfwidth', {{2,5}},... + % % 'ss_cell_size_threshold', {{5,10}},... + % % 'spat_medfilt_enabled', {{0,1}},... + % % 'trim_pixels', {{0.4,0.6}},... + % % 'verbos', {{0,1}},... + % % 'disableGPU', {{1,0}}... + % % ); + % % settingStr = struct(... + % % 'movie_dataset',{{'/1','/Movie','/movie'}},... + % % 'save_to_movie_dir', {{1,0}},... + % % 'spat_linfilt_halfwidth', {{2,5}},... + % % 'ss_cell_size_threshold', {{5,10}},... + % % 'spat_medfilt_enabled', {{0,1}},... + % % 'trim_pixels', {{0.4,0.6}},... + % % 'verbos', {{0,1}},... + % % 'disableGPU', {{1,0}}... + % % ); + + % [h,w,t] = size(inputMovie); + + % opts.max_cell_radius=30; + % opts.min_cell_spacing=5; + % opts.remove_duplicate_cells = 0; + % % Use GPU + % opts.compute_device='gpu'; + + % % This is how to call the function 'partition_helper()' to find out how many partitions are necessary: + % num_parts = partition_helper(h,w,t,opts.min_cell_spacing,opts.max_cell_radius); + + % % Below call returned num_parts=20. We decide to partition x axis to 4, and y axis to 5. This makes 20 parititions overall. + % nPlotsRoot = sqrt(num_parts); + % if nPlotsRoot<2 + % nPlotsRoot = 2; + % end + % integ = fix(nPlotsRoot); + % fract = abs(nPlotsRoot - integ); + % opts.num_partition_y = ceil(nPlotsRoot); + % opts.num_partition_x = floor(nPlotsRoot)+round(fract) + + % min_cell_spacing=3; + % max_cell_radius=10; + % num_partition_x=3; + % num_partitiony=3; + % cell_keep_tolerance=5; + % subtract_background=1; + + % opts.config.diffuse_F=1; + % opts.config.smooth_T = 0; + % opts.config.smooth_F = 0; + % opts.config.cell_keep_tolerance + + % [filters,traces,info,opts] = extractor(movieList{1},opts); + % [filters,traces,info,opts] = extractor(inputMovie,opts); + % outStruct = extractor(inputMovie,opts); + + % switch pcaicaPCsICsSwitchStr + % case 'Subject' + % nPCsnICs = obj.numExpectedSignals.(obj.signalExtractionMethod).(obj.subjectStr{obj.fileNum}) + % case 'Folder' + % nPCsnICs = obj.numExpectedSignals.(obj.signalExtractionMethod).Folders{obj.fileNum} + % otherwise + % % body + % end + % extractConfig.num_estimated_cells = nPCsnICs(1); + + extractConfig.avg_cell_radius = gridWidth.(obj.subjectStr{obj.fileNum}); + extractConfig.preprocess = options.EXTRACT.preprocess; + switch options.EXTRACT.gpuOrCPU + case 'gpu' + extractConfig.use_gpu = 1; + case 'cpu' + extractConfig.use_gpu = 0; + extractConfig.parallel_cpu = 1; + otherwise + % body + end + extractConfig.remove_static_background = false; + extractConfig.skip_dff = true; + + % extractConfig.cellfind_min_snr = options.EXTRACT.cellfind_min_snr; + % extractConfig.num_partitions_x = options.EXTRACT.num_partitions_x; + % extractConfig.num_partitions_y = options.EXTRACT.num_partitions_y; + % extractConfig.compact_output = options.EXTRACT.compact_output; + + % extractConfig.thresholds.T_min_snr = 3; % multiply with noise_std + % extractConfig.thresholds.size_lower_limit = 1/5; % multiply with avg_cell_area + % extractConfig.thresholds.size_upper_limit = 5; % multiply with avg_cell_area + % extractConfig.thresholds.temporal_corrupt_thresh = 0.7; + % extractConfig.thresholds.spatial_corrupt_thresh = 0.7; + % extractConfig.thresholds.T_dup_corr_thresh = 0.95; + % extractConfig.thresholds.S_dup_corr_thresh = 0.95; + % extractConfig.thresholds.eccent_thresh = 6; % set to inf if dendrite aware + % extractConfig.thresholds.low_ST_index_thresh = 1e-2; + % extractConfig.thresholds.high_ST_index_thresh = 0.8; + + + startTime = tic; + outStruct = extractor(inputMovie,extractConfig); + outStruct + + % im_dup_corr_thresh = 0.05; % Image correlation threshold + % trace_dup_corr_thresh = 0.6; % Trace correlation threshold + % outStruct = remove_duplicates(outStruct,im_dup_corr_thresh,trace_dup_corr_thresh); + + extractAnalysisOutput.filters = outStruct.spatial_weights; + % permute so it is [nCells frames] + extractAnalysisOutput.traces = permute(outStruct.temporal_weights, [2 1]); + extractAnalysisOutput.info = outStruct.info; + extractAnalysisOutput.config = outStruct.config; + extractAnalysisOutput.info = outStruct.info; + % Remove the large summary field since takes up unnecessary space + extractAnalysisOutput.info.summary = []; + extractAnalysisOutput.file = movieList{1}; + extractAnalysisOutput.userInputConfig = extractConfig; + % for backwards compatibility + extractAnalysisOutput.opts = outStruct.config; + extractAnalysisOutput.time.startTime = startTime; + extractAnalysisOutput.time.endTime = tic; + extractAnalysisOutput.time.totalTime = toc(startTime); + + % ======= + % save EXTRACT signals + for i=1:length(saveID) + savestring = [thisDirSaveStr saveID{i}]; + display(['saving: ' savestring]) + % save(savestring,saveVariable{i},'-v7.3','emOptions'); + save(savestring,saveVariable{i},'-v7.3'); + end + % ======= + + % Save output in NWB format if requested by user. + subfxnSaveNwbFiles(extractAnalysisOutput.filters,{extractAnalysisOutput.traces}); + end function [cnmfOptions] = runCNMFSignalFinder() % Check CVX is installed and if not, setup. diff --git a/@calciumImagingAnalysis/modelGetSignalsImages.m b/@calciumImagingAnalysis/modelGetSignalsImages.m index b1a5bba..13d745d 100644 --- a/@calciumImagingAnalysis/modelGetSignalsImages.m +++ b/@calciumImagingAnalysis/modelGetSignalsImages.m @@ -103,7 +103,7 @@ fprintf('Search: %s\n',obj.extractionMethodStructSaveStr.(obj.signalExtractionMethod)) options.regexPairs = {{obj.extractionMethodStructSaveStr.(obj.signalExtractionMethod)}}; end - end + end regexPairs = options.regexPairs; @@ -417,29 +417,29 @@ inputSignals = double(emAnalysisOutput.(emOutputType{emI})); break; end - end - - if 0 - if isfield(emAnalysisOutput,'scaledProbability') - disp('Using scaledProbability...') - inputSignals = double(emAnalysisOutput.scaledProbability); - elseif isfield(emAnalysisOutput,'dsScaledProbability') - disp('Using dsScaledProbability...') - inputSignals = double(emAnalysisOutput.dsScaledProbability); - elseif isfield(emAnalysisOutput,'cellTraces') - disp('Using cellTraces...') - inputSignals = double(emAnalysisOutput.cellTraces); - elseif isfield(emAnalysisOutput,'dsCellTraces') - disp('Using dsCellTraces...') - if length(emAnalysisOutput.dsCellTraces)==1 - inputSignals = emAnalysisOutput.cellTraces; - else - inputSignals = emAnalysisOutput.dsCellTraces; - end - else - inputSignals = emAnalysisOutput.cellTraces; - end - end + end + + if 0 + if isfield(emAnalysisOutput,'scaledProbability') + disp('Using scaledProbability...') + inputSignals = double(emAnalysisOutput.scaledProbability); + elseif isfield(emAnalysisOutput,'dsScaledProbability') + disp('Using dsScaledProbability...') + inputSignals = double(emAnalysisOutput.dsScaledProbability); + elseif isfield(emAnalysisOutput,'cellTraces') + disp('Using cellTraces...') + inputSignals = double(emAnalysisOutput.cellTraces); + elseif isfield(emAnalysisOutput,'dsCellTraces') + disp('Using dsCellTraces...') + if length(emAnalysisOutput.dsCellTraces)==1 + inputSignals = emAnalysisOutput.cellTraces; + else + inputSignals = emAnalysisOutput.dsCellTraces; + end + else + inputSignals = emAnalysisOutput.cellTraces; + end + end inputSignals2 = double(emAnalysisOutput.cellTraces); diff --git a/@calciumImagingAnalysis/modelPreprocessMovieFunction.m b/@calciumImagingAnalysis/modelPreprocessMovieFunction.m index fab008a..75b59c4 100644 --- a/@calciumImagingAnalysis/modelPreprocessMovieFunction.m +++ b/@calciumImagingAnalysis/modelPreprocessMovieFunction.m @@ -30,6 +30,7 @@ % 2020.07.07 [00:32:59] - Further upgraded adding json to HDF5 directly for later reading out to get settings. % 2020.09.22 [00:11:03] - Updated to add NWB support. % 2020.10.24 [18:30:56] - Added support for calculating dropped frames if entire frame of a movie is a set value. Changed order so that dropped frames calculated before dF/F. + % 2021.02.15 [12:06:59] - _inputMovieF0 now saved to processing subfolder. % TODO % Insert NaNs or mean of the movie into dropped frame location, see line 260 % Allow easy switching between analyzing all files in a folder together and each file in a folder individually @@ -545,7 +546,8 @@ currentDateTimeStr = datestr(now,'yyyymmdd_HHMMSS','local'); mkdir([thisDir filesep 'processing_info']) thisProcessingDir = [thisDir filesep 'processing_info']; - diarySaveStr = [thisDir filesep 'processing_info' filesep currentDateTimeStr '_preprocess.log']; + thisProcessingDirFileStr = [thisProcessingDir filesep currentDateTimeStr]; + diarySaveStr = [thisProcessingDirFileStr '_preprocess.log']; diary(diarySaveStr); display([num2str(fileNum) '/' num2str(length(folderList)) ': ' thisDir]); @@ -566,6 +568,7 @@ % base string to save as fileInfoSaveStr = [fileInfo.date '_' fileInfo.protocol '_' fileInfo.subject '_' fileInfo.assay]; thisDirSaveStr = [thisDir filesep fileInfoSaveStr]; + thisProcessingDirFileInfoStr = [thisProcessingDir filesep currentDateTimeStr '_' fileInfoSaveStr]; saveStr = ''; % add the folder to the output structure ostruct.folderList{fileNum} = thisDir; @@ -1311,7 +1314,8 @@ function dfofInputMovie() end % Save out F0 in case need later - savePathStr = [thisDirSaveStr '_inputMovieF0' '.h5']; + % savePathStr = [thisDirSaveStr '_inputMovieF0' '.h5']; + savePathStr = [thisProcessingDirFileInfoStr '_inputMovieF0' '.h5']; movieSaved = writeHDF5Data(inputMovieF0,savePathStr,'deflateLevel',options.deflateLevel,'datasetname',options.outputDatasetName); thisMovieMean = nanmean(inputMovieF0(:)); diff --git a/@calciumImagingAnalysis/viewMovie.m b/@calciumImagingAnalysis/viewMovie.m index 7d9fe09..7fd310a 100644 --- a/@calciumImagingAnalysis/viewMovie.m +++ b/@calciumImagingAnalysis/viewMovie.m @@ -11,6 +11,7 @@ % 2019.08.30 [12:59:44] - Added fallback to playMovie in case of Miji error % 2019.10.09 [17:58:22] - Make view movie multi-column % 2020.10.25 [21:05:21] - Added support for viewing movies from disk. + % 2021.02.24 [09:04:24] - Fix treatMoviesAsContinuous=0 + playMovieFromDisk=1 issue. % TODO % % ===================== @@ -105,7 +106,7 @@ '1',... obj.behaviorVideoRegexp,... '0',... - '1',... + '0',... obj.inputDatasetName... '1',... '0',... @@ -216,6 +217,10 @@ [frameListTmp] = getProperFrameList('primary'); if treatMoviesAsContinuous==1 movieListTmp2 = movieList; + % if iscell(movieList) + % else + % movieListTmp2 = {movieList}; + % end else movieListTmp2 = movieList{movieMontageIdx(movieNo)}; end @@ -286,7 +291,11 @@ % ================================================= [frameListTmp] = getProperFrameList('primary'); if treatMoviesAsContinuous==1 - movieListTmp2 = movieList; + if iscell(movieList) + movieListTmp2 = movieList; + else + movieListTmp2 = {movieList}; + end else movieListTmp2 = movieList{movieMontageIdx(movieNo)}; end @@ -294,6 +303,9 @@ if playMovieFromDisk==1 disp('dddd') movieListTmp2 + if ~iscell(movieListTmp2) + movieListTmp2 = {movieListTmp2}; + end try [exitSignal, movieStruct] = playMovie(movieListTmp2{1},'extraTitleText',''); catch err diff --git a/README.md b/README.md index 82ae42e..de5d102 100644 --- a/README.md +++ b/README.md @@ -26,15 +26,16 @@ `CIAtah` features: - A GUI with different modules to allow users to do large-scale batch analysis, accessed via the repository's `ciatah` class. -- The `ciatah` functions can be used to create GUI-less, command line-ready analysis pipelines. Functions are located in `ciapkg` and `+ciapkg` sub-folders. +- The `ciatah` functions can be used to create GUI-less, command line-ready analysis pipelines. Functions are located in the `ciapkg` sub-folder and in the `+ciapkg` package. - Includes all major calcium imaging analysis steps: movie visualization (including reading from disk), pre-processing (motion correction, spatiotemporal downsampling, spatial filtering, relative fluorescence calculation, etc.), support for multiple cell-extraction methods (CELLMax, PCA-ICA, CNMF, CNMF-E, EXTRACT, etc.), manual classification via GUIs, automated cell classification (coming soon!), cross-session cell alignment, and more. - Has several example one- and two-photon calcium imaging datasets that `ciatah` can automatically download to help users test out the package. - Includes code for determining animal position (e.g. in open-field assay). - Supports [Neurodata Without Borders](https://www.nwb.org/) data standard (see [calcium imaging tutorial](https://neurodatawithoutborders.github.io/matnwb/tutorials/html/ophys.html)) for reading/writing cell-extraction (e.g. outputs of PCA-ICA, CELLMax, CNMF, CNMF-E, etc.). Supports reading and writing NWB movie files with continued integration planned. +- Supports most major imaging movie file formats: HDF5, NWB, AVI, ISXD [Inscopix], and TIFF. - Requires `MATLAB`. -Contact: __Biafra Ahanonu, PhD (bahanonu [at] alum [dot] mit [dot] edu)__. +Contact: __Biafra Ahanonu, PhD (github [at] bahanonu [dot] com)__. Made in USA.
USA @@ -109,7 +110,7 @@ Below are steps needed to quickly get started using the `CIAtah` software packag cd('calciumImagingAnalysis-master') ``` -- Run `CIAtah` using the below MATLAB commands. Call `obj;` each time you want to go back to the main GUI. +- Run `CIAtah` using the below MATLAB commands. Call `obj;` in the MATLAB command window each time you want to go back to the main GUI. ```MATLAB % Loads the class into an object for use in this session @@ -128,7 +129,7 @@ obj % then hit enter, no semicolon! ### Visualize any movie quickly using read from disk -Users can quickly visualize movies in any of the supported formats (HDF5, AVI, TIFF, and ISXD) using the `playMovie` function. This will read directly from disk, allowing users to scroll through frames to visually check movies before or after processing. See below code: +Users can quickly visualize movies in any of the supported formats (HDF5, NWB, AVI, TIFF, and ISXD) using the `playMovie` function. This will read directly from disk, allowing users to scroll through frames to visually check movies before or after processing. See below code: ```MATLAB % Use the absolute path to the movie file or a valid relative path. @@ -140,7 +141,6 @@ When using HDF5 files, check the dataset name containing movie with `h5disp` the playMovie('ABSOLUTE\PATH\TO\MOVIE','inputDatasetName','/acquisition/TwoPhotonSeries/data'); ``` - ## Quick start (command line or GUI-less batch analysis) After downloading `CIAtah` and running the setup as above, users interested in command-line processing can open up the example M-file by running the below command. By running individual code-block cells, users are guided from pre-processing through cell-extraction to cross-session analysis. @@ -170,11 +170,11 @@ __Certain sections become available when user selects the appropriate module (e. - See additional details on the [Processing calcium imaging data](https://bahanonu.github.io/calciumImagingAnalysis/pipeline_overview/) page for running the full processing pipeline. - Settings used to pre-process imaging movies (`modelPreprocessMovie` module) are stored inside the processed HDF5 movie to allow `CIAtah` to load them again later. - To force load all directories, including most external software packages (in `_external_programs` folder), type `ciapkg.loadAllDirs;` into MATLAB command line. This is most relevant when you need to access specific functions in an outside repository that are normally hidden until needed. -- When issues are encountered, first check the [Common issues and fixes](https://bahanonu.github.io/calciumImagingAnalysis/help_issues/) page to see if a solution is there. Else, submit a new issue or email Biafra (bahanonu [at] alum [dot] mit [dot] edu). +- When issues are encountered, first check the [Common issues and fixes](https://bahanonu.github.io/calciumImagingAnalysis/help_issues/) page to see if a solution is there. Else, submit a new issue or email Biafra. - There are two sets of test data that are downloaded: - __Single session analysis__: `data\2014_04_01_p203_m19_check01_raw` can be used to test the pipeline until the cross-session alignment step. - __Batch analysis__: `data\batch` contains three imaging sessions that should be processed and can then be used for the cross-session alignment step. Users should try these sessions to get used to batched analysis. -- For Fiji dependency, when path to `Miji.m` (e.g. `\Fiji.app\scripts` folder) is requested, likely in `calciumImagingAnalysis\_external_programs\FIJI_FOLDER\Fiji.app\scripts` where `FIJI_FOLDER` varies depending on OS, unless the user requested a custom path or on OSX (in which case, find Fiji the install directory). +- For Fiji dependency, when path to `Miji.m` (e.g. `\Fiji.app\scripts` folder) is requested, likely in `[CIAtah directory]\_external_programs\FIJI_FOLDER\Fiji.app\scripts` where `FIJI_FOLDER` varies depending on OS, unless the user requested a custom path or on OSX (in which case, find Fiji the install directory). - If you run into Java heap space memory errors when Miji tries to load Fiji in MATLAB, make sure "java.opts" file is in MATLAB start-up folder or that the `CIAtah` root folder is the MATLAB start-up folder ([instructions on changing](https://www.mathworks.com/help/matlab/matlab_env/matlab-startup-folder.html)). - `CIAtah` often uses [regular expressions](https://www.cheatography.com/davechild/cheat-sheets/regular-expressions/) to find relevant movie and other files in folders to analyze. - For example, by default it looks for any movie files in a folder containing `concat`, e.g. `concat_recording_20140401_180333.h5` (test data). If you have a file called `rawData_2019_01_01_myInterestingExperiment.avi` and all your raw data files start with `rawData_` then change the regular expression to `rawData_` when requested by the repository. See `setMovieInfo` module to change after adding new folders. @@ -245,7 +245,7 @@ Please cite [Corder*, Ahanonu*, et al. 2019](http://science.sciencemag.org/conte ``` ## Questions? -Please email any additional questions not covered in the repository to `bahanonu [at] alum [dot] mit [dot] edu` or open an issue. +Please email any additional questions not covered in the repository to `github [at] bahanonu [dot] com` or open an issue. *** diff --git a/ciapkg/VERSION b/ciapkg/VERSION index 9a8c76a..fa8dfe1 100644 --- a/ciapkg/VERSION +++ b/ciapkg/VERSION @@ -1,2 +1,2 @@ -v3.23.1 -20210121172228 \ No newline at end of file +v3.24.1 +20210221190557 \ No newline at end of file diff --git a/ciapkg/behavior/computeSaleaeOutput.m b/ciapkg/behavior/computeSaleaeOutput.m index ec6fede..81f271a 100644 --- a/ciapkg/behavior/computeSaleaeOutput.m +++ b/ciapkg/behavior/computeSaleaeOutput.m @@ -1,5 +1,5 @@ function [outputData] = computeSaleaeOutput(matfile,varargin) - % Processes Saleae output files. + % Processes Saleae output files. This is for data collected with Saleae Logic 1.x software, NOT 2.x. % Biafra Ahanonu % started: 2014.01.03 [19:13:01] % inputs diff --git a/ciapkg/download/downloadCnmfGithubRepositories.m b/ciapkg/download/downloadCnmfGithubRepositories.m index e9cee42..f93aa33 100644 --- a/ciapkg/download/downloadCnmfGithubRepositories.m +++ b/ciapkg/download/downloadCnmfGithubRepositories.m @@ -6,9 +6,10 @@ % 2020.04.03 [14:02:33] - Save downloaded compressed files (e.g. zips) to a sub-folder. % 2020.06.28 [13:08:16] - Final implementation of force update, to bring to most current version of all git directories. % 2020.06.28 [14:01:17] - Switch to calling downloadGithubRepositories for downloads to prevent bugs introduced by similar code between two functions. + % 2021.02.01 [‏‎15:19:40] - Update `_external_programs` to call ciapkg.getDirExternalPrograms() to standardize call across all functions. %======================== - options.defaultExternalProgramDir = ['_external_programs']; + options.defaultExternalProgramDir = ciapkg.getDirExternalPrograms(); % 1 = force update of the git repository, 0 = skip if already downloaded options.forceUpdate = 0; % options.downloadPreprocessed = 0; diff --git a/ciapkg/download/downloadGithubRepositories.m b/ciapkg/download/downloadGithubRepositories.m index 225be1a..6c18ca0 100644 --- a/ciapkg/download/downloadGithubRepositories.m +++ b/ciapkg/download/downloadGithubRepositories.m @@ -7,12 +7,13 @@ % 2020.04.03 [14:02:33] - Save downloaded compressed files (e.g. zips) to a sub-folder. % 2020.06.28 [13:08:16] - Final implementation of force update, to bring to most current version of all git directories. % 2021.01.22 [13:18:25] - Update to allow regexp backup to find name of downloaded Github repo folder after unzipping, e.g. in cases where a release or non-master branch is downloaded. - IGNORE + % 2021.02.01 [‏‎15:19:40] - Update `_external_programs` to call ciapkg.getDirExternalPrograms() to standardize call across all functions. %======================== % 1 = force update of the git repository, 0 = skip if already downloaded options.forceUpdate = 0; % Str: directory of external download path. - options.signalExtractionDir = '_external_programs'; + options.signalExtractionDir = ciapkg.getDirExternalPrograms(); options.gitNameDisp = {'NoRMCorre'}; options.gitRepos = {'https://github.com/flatironinstitute/NoRMCorre/archive/master.zip'}; options.outputDir = {'normcorre'}; diff --git a/ciapkg/download/downloadMiji.m b/ciapkg/download/downloadMiji.m index d248106..630944e 100644 --- a/ciapkg/download/downloadMiji.m +++ b/ciapkg/download/downloadMiji.m @@ -10,12 +10,13 @@ % changelog % 2019.10.15 [10:48:10] - Fix so DMGs downloaded for MAC have the proper file extension. % 2020.04.03 [14:02:33] - Save downloaded compressed files (e.g. zips) to a sub-folder. + % 2021.02.01 [‏‎15:19:40] - Update `_external_programs` to call ciapkg.getDirExternalPrograms() to standardize call across all functions. % TODO % %======================== % options.defaultDir = ['private' filesep 'programs']; - options.defaultDir = ['_external_programs']; + options.defaultDir = ciapkg.getDirExternalPrograms(); % get options options = getOptions(options,varargin); % display(options) diff --git a/ciapkg/download/example_downloadTestData.m b/ciapkg/download/example_downloadTestData.m index 7b3c89c..7e20e5d 100644 --- a/ciapkg/download/example_downloadTestData.m +++ b/ciapkg/download/example_downloadTestData.m @@ -1,5 +1,5 @@ function [success] = example_downloadTestData(varargin) - % Downloads example test data from Stanford Box + % Downloads CIAtah example test data from online into data folder. % Biafra Ahanonu % started: September 2018 % inputs @@ -10,6 +10,7 @@ % changelog % 2019.09.16 [13:03:33] - Added three new imaging sessions to use for cross-day alignment and made downloading more generalized. % 2020.09.14 [13:17:57] - Added example two-photon dataset. + % 2021.02.02 [11:25:34] - Function now calls data directory via standardized ciapkg.getDirPkg('data') to avoid placing data in incorrect folder. % TODO % @@ -17,6 +18,8 @@ options.downloadPreprocessed = 0; % Download extra files options.downloadExtraFiles = 1; + % Directory where download folder goes + options.dataDir = ciapkg.getDirPkg('data'); % get options options = getOptions(options,varargin); % display(options) @@ -76,7 +79,7 @@ for fileNo = 1:nFiles fileInfo = downloadList{fileNo}; - rawSavePathDownload = ['data' filesep fileInfo.folderName]; + rawSavePathDownload = [options.dataDir filesep fileInfo.folderName]; if ~exist(rawSavePathDownload,'dir');mkdir(rawSavePathDownload);fprintf('Made folder: %s\n',rawSavePathDownload);end rawSavePathDownload = [rawSavePathDownload filesep fileInfo.fileName]; @@ -108,7 +111,7 @@ % end if options.downloadPreprocessed==1 - rawSavePathDownload = ['data' filesep '2014_04_01_p203_m19_check01'] + rawSavePathDownload = [options.dataDir filesep '2014_04_01_p203_m19_check01'] if ~exist(rawSavePathDownload,'dir');mkdir(rawSavePathDownload);fprintf('Made folder: %s',rawSavePathDownload);end rawSavePathDownload = [rawSavePathDownload filesep '2014_04_01_p203_m19_check01_turboreg_crop_dfof_downsampleTime_1.h5']; @@ -117,7 +120,7 @@ websave(rawSavePathDownload,'https://stanford.box.com/shared/static/0zasceqd7b9ea6pa4rsgx1ag1mpjwmrf.h5'); end - rawSavePathDownload = ['data' filesep '2014_04_01_p203_m19_check01' filesep '2014_04_01_p203_m19_check01_turboreg_crop_dfof_1.h5']; + rawSavePathDownload = [options.dataDir filesep '2014_04_01_p203_m19_check01' filesep '2014_04_01_p203_m19_check01_turboreg_crop_dfof_1.h5']; if exist(rawSavePathDownload,'file')~=2 fprintf('Downloading file to %s\n',rawSavePathDownload) websave(rawSavePathDownload,'https://stanford.box.com/shared/static/azabf70oky7vriek48pb98jt2c5upj5i.h5'); diff --git a/ciapkg/hdf5/createHdf5File.m b/ciapkg/hdf5/createHdf5File.m index 968b376..bcbb208 100644 --- a/ciapkg/hdf5/createHdf5File.m +++ b/ciapkg/hdf5/createHdf5File.m @@ -10,6 +10,7 @@ function createHdf5File(filename, datasetName, inputData, varargin) % changelog % 2019.03.25 [17:17:49] - Add support for custom user HDF5 chunking as opposed to previous automatic chunking % 2019.08.20 [11:38:54] - Added additional support for more data types. + % 2021.02.02 [13:15:11] - Close space_id, dset_id, and fid with low-level HDF5 functions before appending data with hdf5write to avoid read/write issues. % TODO % %======================== @@ -87,7 +88,13 @@ function createHdf5File(filename, datasetName, inputData, varargin) % Write the initial data H5D.write(dset_id, 'H5ML_DEFAULT', 'H5S_ALL', 'H5S_ALL', 'H5P_DEFAULT', inputData); + + % Close the opened identifiers + H5S.close(space_id); + H5D.close(dset_id); + H5F.close(fid); + % Append movie relevant information to HDF5 file % if strcmp(options.writeMode,'new') % hdf5write(filename,'/movie/info/dimensions',dataDims,'WriteMode','append'); currentDateTimeStr = datestr(now,'yyyymmdd_HHMM','local'); @@ -96,11 +103,6 @@ function createHdf5File(filename, datasetName, inputData, varargin) hdf5write(filename,'/movie/info/Deflate',options.deflateLevel,'WriteMode','append'); % end - % Close the open Identifiers - H5S.close(space_id); - H5D.close(dset_id); - H5F.close(fid); - % add information about data to HDF5 file if ~isempty(options.addInfo) if ~iscell(options.addInfo) diff --git a/ciapkg/hdf5/readHDF5Subset.m b/ciapkg/hdf5/readHDF5Subset.m index 48e4e1f..0074efb 100644 --- a/ciapkg/hdf5/readHDF5Subset.m +++ b/ciapkg/hdf5/readHDF5Subset.m @@ -19,6 +19,7 @@ % 2019.02.13 [17:57:55] - Improved duplicate frame support, finds differences in frames, loads all unique as a single slab, then loads remaining and re-organizes to be in correct order. % 2019.05.03 [15:42:08] - Additional 4D support in cases where a 3D offset/block request is made. % 2019.10.10 [12:52:54] - Add correction for frame order. Select hyperslab in HDF5 makes blocks in sorted order, so after reading the explicit offset ordering is not the original unsorted order. + % 2021.02.15 [12:02:36] - Updated support for files with datasets that contain 2D matrices. % TODO % DONE: Make support for duplicate frames more robust so minimize the number of file reads. @@ -81,7 +82,9 @@ dset_id = H5D.open(fid,options.datasetName); dims = fliplr(block{1});%[xDim yDim 1] tmpVar = sum(cat(1,block{:}),1); - if length(block{1})==3 + if length(block{1})==2 + % Do nothing. + elseif length(block{1})==3 dims(1) = tmpVar(3); elseif length(block{1})==4 dims(1) = tmpVar(4); diff --git a/ciapkg/io/loadMovieList.m b/ciapkg/io/loadMovieList.m index 3457f47..fed47c0 100644 --- a/ciapkg/io/loadMovieList.m +++ b/ciapkg/io/loadMovieList.m @@ -49,6 +49,8 @@ % 2020.08.30 [10:16:08] - Change warning message output for HDF5 file of certain type. % 2020.08.31 [15:47:49] - Add option to suppress warnings. % 2020.10.19 [12:11:14] - Improved comments and options descriptions. + % 2021.02.15 [11:55:36] - Fixed loading HDF5 datasetname that has only a single frame, loadMovieList would ask for 3rd dimension information that did not exist. + % TODO % OPEN % Determine file type by properties of file instead of extension (don't trust input...) @@ -255,16 +257,30 @@ hReadInfo.Dims = datasetDims; dims.x(iMovie) = hReadInfo.Dims(1); dims.y(iMovie) = hReadInfo.Dims(2); - dims.z(iMovie) = hReadInfo.Dims(3); dims.one(iMovie) = hReadInfo.Dims(1); dims.two(iMovie) = hReadInfo.Dims(2); - dims.three(iMovie) = hReadInfo.Dims(3); + % Check 3rd dimension exists + if length(hReadInfo.Dims)>=3 + dims.z(iMovie) = hReadInfo.Dims(3); + dims.three(iMovie) = hReadInfo.Dims(3); + else + dims.z(iMovie) = 0; + dims.three(iMovie) = 0; + end + if dims.z(iMovie) + offsetTmp = [0 0 1]; + blockTmp = [dims.x(iMovie) dims.y(iMovie) 1]; + else + offsetTmp = [0 0]; + blockTmp = [dims.x(iMovie) dims.y(iMovie)]; + end if ischar(options.inputDatasetName) - tmpFrame = readHDF5Subset(thisMoviePath,[0 0 1],[dims.x(iMovie) dims.y(iMovie) 1],'datasetName',options.inputDatasetName,'displayInfo',options.displayInfo); + tmpDataset = options.inputDatasetName; else - tmpFrame = readHDF5Subset(thisMoviePath,[0 0 1],[dims.x(iMovie) dims.y(iMovie) 1],'datasetName',thisDatasetName,'displayInfo',options.displayInfo); + tmpDataset = thisDatasetName; end + tmpFrame = readHDF5Subset(thisMoviePath,offsetTmp,blockTmp,'datasetName',tmpDataset,'displayInfo',options.displayInfo); case 'avi' xyloObj = VideoReader(thisMoviePath); dims.x(iMovie) = xyloObj.Height; @@ -674,7 +690,11 @@ end if exist('tmpMovie','var') if iMovie==1 - outputMovie(1:dims.x(iMovie),1:dims.y(iMovie),1:dims.z(iMovie)) = tmpMovie; + if dims.z(iMovie)==0 + outputMovie(1:dims.x(iMovie),1:dims.y(iMovie),1) = tmpMovie; + else + outputMovie(1:dims.x(iMovie),1:dims.y(iMovie),1:dims.z(iMovie)) = tmpMovie; + end % outputMovie(:,:,:) = tmpMovie; else % assume 3D movies with [x y frames] as dimensions diff --git a/ciapkg/io/loadNeurodataWithoutBorders.m b/ciapkg/io/loadNeurodataWithoutBorders.m index 7368973..6155355 100644 --- a/ciapkg/io/loadNeurodataWithoutBorders.m +++ b/ciapkg/io/loadNeurodataWithoutBorders.m @@ -1,4 +1,4 @@ -function [inputImages,inputTraces,infoStruct] = loadNeurodataWithoutBorders(inputFilePath,varargin) +function [inputImages,inputTraces,infoStruct, algorithmStr] = loadNeurodataWithoutBorders(inputFilePath,varargin) % DESCRIPTION. % Biafra Ahanonu % started: 2020.04.04 [15:02:22] @@ -8,7 +8,7 @@ % % changelog - % + % 2021.02.05 [19:02:44] - Parse the algorithm associated with the NWB signal extraction data. % TODO % @@ -44,6 +44,7 @@ inputImages = []; inputTraces = []; infoStruct = struct; + algorithmStr = ''; if iscell(inputFilePath) inputFilePath = inputFilePath{1}; @@ -101,6 +102,19 @@ % Get description if later need cell-extraction information tmp = h5readatt(inputFilePath,imagesGroupNameAttr,'description'); infoStruct.description = tmp; + + % Check if extraction method in infoStruct then add + try + if isfield(infoStruct,'description') + tmpStr = regexp(infoStruct.description,'Extraction method: \w+','match'); + tmpStr = strsplit(tmpStr{1}{1},': '); + algorithmStr = tmpStr{2}; + end + catch err + disp(repmat('@',1,7)) + disp(getReport(err,'extended','hyperlinks','on')); + disp(repmat('@',1,7)) + end catch err disp(repmat('@',1,7)) disp(getReport(err,'extended','hyperlinks','on')); diff --git a/ciapkg/io/modelAddOutsideDependencies.m b/ciapkg/io/modelAddOutsideDependencies.m index 703c716..e67cea7 100644 --- a/ciapkg/io/modelAddOutsideDependencies.m +++ b/ciapkg/io/modelAddOutsideDependencies.m @@ -9,12 +9,13 @@ % changelog % 2019.10.15 [12:29:30] - Added flag to prevent recursive loop between resetMiji and modelAddOutsideDependencies. + % 2021.02.01 [‏‎15:19:40] - Update `_external_programs` to call ciapkg.getDirExternalPrograms() to standardize call across all functions. % TODO % %======================== options.exampleOption = ''; - options.defaultExternalProgramDir = ['_external_programs']; + options.defaultExternalProgramDir = ciapkg.getDirExternalPrograms(); options.recursionExit = 0; % get options options = getOptions(options,varargin); diff --git a/ciapkg/io/saveNeurodataWithoutBorders.m b/ciapkg/io/saveNeurodataWithoutBorders.m index 364f671..145eae0 100644 --- a/ciapkg/io/saveNeurodataWithoutBorders.m +++ b/ciapkg/io/saveNeurodataWithoutBorders.m @@ -5,7 +5,7 @@ % Based on mat2nwb in https://github.com/schnitzer-lab/nwb_schnitzer_lab. % inputs % image_masks - [x y z] matrix - % roi_response_data - {1 N} cell with N = number of different signal traces for that algorithm. + % roi_response_data - {1 N} cell with N = number of different signal traces for that algorithm. Make sure each signal trace matrix is in form of [nSignals nFrames]. % algorithm - Name of the algorithm. % outputFilePath - file path to save NWB file to. % outputs @@ -14,12 +14,15 @@ % changelog % 2020.07.01 [09:40:20] - Convert roi_response_data to cell if user inputs only a matrix. % 2020.09.15 [20:30:32] - Automatically creates directory where file is to be stored if it is not present. + % 2021.02.01 [?‎15:14:40] - Function checks that yaml, matnwb, and nwb_schnitzer_lab loaded, else tries to load to make sure all dependencies are present and active. + % 2021.02.01 [15:19:40] - Update `_external_programs` to call ciapkg.getDirExternalPrograms() to standardize call across all functions. + % 2021.02.03 [12:34:06] - Added a check for inputs with a single signal and function returns as it is not supported. % TODO % %======================== % DESCRIPTION - options.fpathYML = [ciapkg.getDir filesep '_external_programs' filesep 'nwb_schnitzer_lab' filesep 'ExampleMetadata.yml']; + options.fpathYML = [ciapkg.getDirExternalPrograms() filesep 'nwb_schnitzer_lab' filesep 'ExampleMetadata.yml']; % get options options = getOptions(options,varargin); % display(options) @@ -30,8 +33,39 @@ % end %======================== + success = 0; + + try + % Check that all necessary files are loaded + loadDependenciesFlag = 0; + if length(which('yaml.ReadYaml'))==0 + disp('yaml not loaded, loading now...') + loadDependenciesFlag = 1; + end + if length(which('get_input_args'))==0 + disp('matnwb not loaded, loading now...') + loadDependenciesFlag = 1; + end + if length(which('add_processed_ophys'))==0 + disp('nwb_schnitzer_lab not loaded, loading now...') + loadDependenciesFlag = 1; + end + if loadDependenciesFlag==1 + ciapkg.io.loadDependencies(... + 'guiEnabled',0,... + 'depIdxArray',5,... + 'forceUpdate',0); + % 'dependencyStr','downloadNeuroDataWithoutBorders',... + % 'dispStr','Download NWB (NeuroDataWithoutBorders)',... + ciapkg.loadDirs; + end + catch err + disp(repmat('@',1,7)) + disp(getReport(err,'extended','hyperlinks','on')); + disp(repmat('@',1,7)) + end + try - success = 0; metadata = yaml.ReadYaml(options.fpathYML); data_path = outputFilePath; @@ -72,6 +106,10 @@ tmpData = roi_response_data; roi_response_data = struct; for i=1:length(tmpData) + if size(1,tmpData{i})==1 + disp('Only a single output, NWB will not support at the moment.') + return; + end roi_response_data.(['ROI_' num2str(i)]) = tmpData{i}; end diff --git a/ciapkg/signal_extraction/cnmfVersionDirLoad.m b/ciapkg/signal_extraction/cnmfVersionDirLoad.m index 2270fc3..e2620d2 100644 --- a/ciapkg/signal_extraction/cnmfVersionDirLoad.m +++ b/ciapkg/signal_extraction/cnmfVersionDirLoad.m @@ -11,12 +11,13 @@ % 2019.01.23 [09:14:54] - Added support for Matlab versions without `contains` function. % 2019.03.03 [20:58:33] - Added removal of cvx from path since they overload `narginchk` which can cause warnings. % 2019.11.13 [18:05:12] - Updated to make contains not include less than 9.1. + % 2021.02.01 [‏‎15:19:40] - Update `_external_programs` to call ciapkg.getDirExternalPrograms() to standardize call across all functions. % TODO % %======================== % Relative path assumed for batch_processing package - options.signalExtractionRootPath = '_external_programs'; + options.signalExtractionRootPath = ciapkg.getDirExternalPrograms(); % Binary: 1 = display paths to be added or removed options.displayOutput = 1; % get options diff --git a/ciapkg/signal_extraction/computeCnmfeSignalExtraction_batch.m b/ciapkg/signal_extraction/computeCnmfeSignalExtraction_batch.m index ce7baaf..2dfaa51 100644 --- a/ciapkg/signal_extraction/computeCnmfeSignalExtraction_batch.m +++ b/ciapkg/signal_extraction/computeCnmfeSignalExtraction_batch.m @@ -18,6 +18,7 @@ % changelog % 2016.06.20 - updated to keep in line with recent changes to CNMF functions + % 2021.01.24 [14:29:06] - Added trace origin type to output structure. % TODO % @@ -385,6 +386,8 @@ cnmfeAnalysisOutput.extractedImages = reshape(full(results.A),[neuron.options.d1 neuron.options.d2 size(results.C,1)]); cnmfeAnalysisOutput.extractedSignals = results.C; cnmfeAnalysisOutput.extractedSignalsEst = results.C_raw; + cnmfAnalysisOutput.extractedSignalsType = 'model'; + cnmfAnalysisOutput.extractedSignalsEstType = 'dfof'; cnmfeAnalysisOutput.extractedPeaks = results.S; cnmfeAnalysisOutput.Cn = results.Cn; cnmfeAnalysisOutput.P = results.P; diff --git a/ciapkg/signal_extraction/runCvxSetup.m b/ciapkg/signal_extraction/runCvxSetup.m index 24d6f21..fe0da7c 100644 --- a/ciapkg/signal_extraction/runCvxSetup.m +++ b/ciapkg/signal_extraction/runCvxSetup.m @@ -8,13 +8,13 @@ % % changelog - % + % 2021.02.01 [‏‎15:19:40] - Update `_external_programs` to call ciapkg.getDirExternalPrograms() to standardize call across all functions. % TODO % %======================== % Str: default location of external programs - options.defaultExternalProgramDir = ['_external_programs']; + options.defaultExternalProgramDir = ciapkg.getDirExternalPrograms(); % get options options = getOptions(options,varargin); % display(options) diff --git a/ciapkg/view/playMovie.m b/ciapkg/view/playMovie.m index 9fff116..852acb8 100644 --- a/ciapkg/view/playMovie.m +++ b/ciapkg/view/playMovie.m @@ -1,5 +1,5 @@ function [exitSignal, ostruct] = playMovie(inputMovie, varargin) - % Plays a 3D matrix as a movie, additional inputs to view multiple movies or sync'd signal data; can also save the resulting figure as a movie. + % Plays a movie that is either a 3D xyt matrix or path to file. Additional inputs to view multiple movies or sync'd signal data and can also save the resulting figure as a movie. % Biafra Ahanonu % started 2013.11.09 [10:39:50] % @@ -28,6 +28,7 @@ % 2020.10.19 [12:51:00] - Switch read from disk support to using ciapkg.io.readFrame so that playMovie uses the new standard interface for fast reading from disk. % 2021.01.14 [20:12:10] - Update passing of HDF5 dataset name to ciapkg.io.readFrame. % 2021.01.17 [19:21:12] - Integrated contrast sliders directly into the main GUI so users don't have to open up a separate GUI. Make GUI sliders thinner. + % 2021.02.05 [16:25:12] - Added feature to sub-sample movie to make display run faster for larger movies. % ======================== % options @@ -39,6 +40,8 @@ % To get around issues with Matlab drawing too fast to detect key strokes, set to 60 or below. options.fpsMax = 80; options.fpsMin = 1/10; + % Int: By what amount to sub-sample the frames in spatial dimensions. 1 = no sub-sampling. >1 = sub-sampling enabled. + options.subSampleMovie = 1; % Matrix: [X Y Z], additional movie to show. X,Y height/width and Z frames. options.extraMovie = []; % 2D matrix: [signals frames] signals related to inputMovie. @@ -128,6 +131,11 @@ [movieTypeExtra, supported, movieTypeSpecificExtra] = ciapkg.io.getMovieFileType(options.extraMovie); end + % If NWB, change dataset name to NWB default + if strcmp(movieTypeSpecific,'nwb') + options.inputDatasetName = options.defaultNwbDatasetName{1}; + end + if supported==0 disp('Unsupported movie type provided.') return; @@ -271,7 +279,8 @@ axis equal tight end - xAxisHandle = xlabel(['frame: 1/' num2str(nFrames) ' fps: ' num2str(options.fps)]); + xAxisHandle = xlabel(sprintf('frame: 1/%d | fps: %d | sub-sample: %d.',nFrames,options.fps,options.subSampleMovie)); + if colorbarsOn==1 set(gcf,'SizeChangedFcn',@(hObject,event) resizeui(hObject,event,axHandle,colorbarsOn)); @@ -510,7 +519,12 @@ end montageHandle = findobj(axHandle,'Type','image'); - set(montageHandle,'Cdata',thisFrame,'AlphaData',imAlpha); + if options.subSampleMovie>1 + ssm = options.subSampleMovie; + set(montageHandle,'Cdata',thisFrame(1:ssm:end,1:ssm:end),'AlphaData',imAlpha(1:ssm:end,1:ssm:end)); + else + set(montageHandle,'Cdata',thisFrame,'AlphaData',imAlpha); + end if strcmp(options.colormapColor,'gray') set(axHandle,'color',[1 0 0]); else @@ -545,11 +559,12 @@ if pauseLoop==1 pauseLoopStr = 'Paused! '; end + % sprintf('frame: 1/%d | fps: %d | sub-sample: %d.',nFrames,options.fps,options.subSampleMovie); if isempty(options.frameList) - tmpStrH = sprintf('%sframe: %d/%d fps: %d %s',pauseLoopStr,frame,nFrames,options.fps,movieDimStr); + tmpStrH = sprintf('%sframe: %d/%d | fps: %d | sub-sample: %d | %s',pauseLoopStr,frame,nFrames,options.fps,options.subSampleMovie,movieDimStr); set(xAxisHandle,'String',tmpStrH); else - tmpStrH = sprintf('%sframe: %d/%d (%d/%d) fps: %d %d',pauseLoopStr,frame,nFrames,options.frameList(frame),nFramesOriginal,options.fps,movieDimStr); + tmpStrH = sprintf('%sframe: %d/%d (%d/%d) | fps: %d %d | sub-sample: %d | %s',pauseLoopStr,frame,nFrames,options.frameList(frame),nFramesOriginal,options.fps,options.subSampleMovie,movieDimStr); set(xAxisHandle,'String',tmpStrH); end catch @@ -996,6 +1011,12 @@ function subfxn_respondUserInput(keyInTmp) end switch double(keyIn) + case 50 % increase sub-sample + options.subSampleMovie = options.subSampleMovie + 1; + case 49 % decrease sub-sample + if options.subSampleMovie>1 + options.subSampleMovie = options.subSampleMovie-1; + end case 105 subfxn_imageJ(inputMovie) case 119 %'w' %set frame rate @@ -1365,6 +1386,8 @@ function subfxn_displayShortcuts(src,event) {'left arrow','','-1 frame'},... sepMenuLins,... {'Note','','Click below LETTER commands to run or use the keyboard shortcut in main UI.'},... + {'1','1','Decrease sub-sample'},... + {'2','2','Increase sub-sample'},... {'E','e','exit'},... {'Q','q','exit all'},... {'G','g','goto frame'},... @@ -1399,10 +1422,9 @@ function subfxn_displayShortcuts(src,event) supTitleStr = ['\texttt{',sepVar,... '\underline{Mouse scroll wheel}: Forward/back frame',titleSep,... '\underline{Mouse middle-click}: pause/play movie',sepVar,... - '\underline{+}:speed',titleSep,... - '\underline{-}:slow',titleSep,... - '\underline{right arrow}:+1 frame',titleSep,... - '\underline{left arrow}:-1 frame',sepVar,... + '\underline{+/-}:Increase/decrease fps',titleSep,... + '\underline{Right/left arrows}:+1 or -1 frame',sepVar,... + '\underline{1/2}: Decrease/Increase movie sub-sample',titleSep,... '\underline{E}: Exit}',... options.extraTitleText,... '']; diff --git a/ciapkg/view/plotSignalsGraph.m b/ciapkg/view/plotSignalsGraph.m index 3035da7..ecb1b16 100644 --- a/ciapkg/view/plotSignalsGraph.m +++ b/ciapkg/view/plotSignalsGraph.m @@ -1,4 +1,4 @@ -function plotSignalsGraph(IcaTraces,varargin) +function [tmpTrace] = plotSignalsGraph(IcaTraces,varargin) % Plots signals, offsetting by a fixed amount. % Biafra Ahanonu % started: 2013.11.02 @@ -9,6 +9,7 @@ function plotSignalsGraph(IcaTraces,varargin) % changelog % 2019.04.22 [19:14:47] - changed from plot to line so when exporting to illustrator don't need to merge lines. % 2019.12.24 [11:20:14] - Allow + % 2021.01.24 [10:08:13] - Function outputs the modified traces for parent functions to use for additional plotting behavior. % TODO % add options for how much to offset @@ -114,8 +115,8 @@ function plotSignalsGraph(IcaTraces,varargin) % plot(tmpTrace','LineWidth',options.LineWidth); plotXaxis = 1:nXaxisPoints; else - display('================') - display('custom x-axis') + disp('================'); + disp('custom x-axis'); % plot(options.inputXAxis,tmpTrace','LineWidth',options.LineWidth); % line(options.inputXAxis,tmpTrace','LineWidth',options.LineWidth); plotXaxis = options.inputXAxis; @@ -133,6 +134,8 @@ function plotSignalsGraph(IcaTraces,varargin) box off; set(groot,'defaultAxesColorOrder',originalAxisColorOrder); + + tmpTrace = flipdim(tmpTrace,1); % for i=1:size(normalTrace,1) % figure(42) diff --git a/ciapkgRoot.m b/ciapkgRoot.m index 3834ee5..b8705c2 100644 --- a/ciapkgRoot.m +++ b/ciapkgRoot.m @@ -1,5 +1,5 @@ function ciapkgRoot() - % Empty function, used to quickly find the root calciumImagingAnalysis folder + % Empty function, used to quickly find the root CIAtah folder % Biafra Ahanonu % started: INSERT_DATE % inputs diff --git a/loadBatchFxns.m b/loadBatchFxns.m index 16e2eb5..24408e7 100644 --- a/loadBatchFxns.m +++ b/loadBatchFxns.m @@ -20,13 +20,14 @@ function loadBatchFxns(varargin) % 2020.05.09 [16:40:13] - Updates to remove additional specific repositories that should not be loaded by default. Add support for overriding this feature. % 2020.06.05 [23:35:43] - If user doesn't have Parallel Toolbox, still works % 2020.07.21 [14:11:42] - Fix to make sure all sub-folders (not just the root) are also removed in the case of certain external_programs. + % 2021.02.01 [‏‎15:19:40] - Update `_external_programs` to call ciapkg.getDirExternalPrograms() to standardize call across all functions. % TODO % % Disable the handle graphics warning "The DrawMode property will be removed in a future release. Use the SortMethod property instead." from being displayed. Comment out this line for debugging purposes as needed. warning('off','MATLAB:hg:WillBeRemovedReplaceWith') - externalProgramsDir = '_external_programs'; + externalProgramsDir = ciapkg.getDirExternalPrograms(); % Add calciumImagingAnalysis directory and subdirectories to path, use dbstack to ensure only add in the root directory regardless of where user has current MATLAB folder. functionLocation = dbstack('-completenames'); @@ -146,7 +147,7 @@ function loadBatchFxns(varargin) else pathtoMiji = [fijiList{1} filesep 'Fiji.app' filesep 'scripts']; end - % pathtoMiji = ['_external_programs' filesep 'fiji-win64-20151222' filesep 'Fiji.app' filesep 'scripts']; + % pathtoMiji = [ciapkg.getDirExternalPrograms() filesep 'fiji-win64-20151222' filesep 'Fiji.app' filesep 'scripts']; % else % end