From 33d2d50ceaad6a877d8a27a8929592e573ff9639 Mon Sep 17 00:00:00 2001 From: George Stuyt Date: Tue, 14 May 2024 15:28:11 +1000 Subject: [PATCH 01/20] Update BpodSystem.Path.CurrentProtocol on protocol selection --- Functions/Launch manager/NewLaunchManager.m | 1 + 1 file changed, 1 insertion(+) diff --git a/Functions/Launch manager/NewLaunchManager.m b/Functions/Launch manager/NewLaunchManager.m index 124e9a99..d3d189b0 100644 --- a/Functions/Launch manager/NewLaunchManager.m +++ b/Functions/Launch manager/NewLaunchManager.m @@ -753,6 +753,7 @@ function LaunchProtocol(a,b) end ProtocolFolderPath = fullfile(BpodSystem.Path.ProtocolFolder,ProtocolName); ProtocolPath = fullfile(BpodSystem.Path.ProtocolFolder,ProtocolName,[ProtocolName '.m']); +BpodSystem.Path.CurrentProtocol = ProtocolPath; addpath(ProtocolFolderPath); set(BpodSystem.GUIHandles.RunButton, 'cdata', BpodSystem.GUIData.PauseButton, 'TooltipString', 'Press to pause session'); IsOnline = BpodSystem.check4Internet(); From 221d2d9f83c2638171ce13dae239cf088951df3c Mon Sep 17 00:00:00 2001 From: George Stuyt Date: Tue, 14 May 2024 15:28:48 +1000 Subject: [PATCH 02/20] Add protocol path and hash to Data.Info --- .../+BpodLib/+External/+DataHash/DataHash.m | 534 ++++++++++++++++++ .../+BpodLib/+External/+DataHash/license.txt | 25 + .../+External/+DataHash/uTest_DataHash.m | 367 ++++++++++++ Functions/+BpodLib/+Launcher/Contents..m | 3 + Functions/+BpodLib/+Launcher/HashFile.m | 16 + Functions/+BpodLib/Contents.m | 4 + Functions/AddTrialEvents.m | 3 + 7 files changed, 952 insertions(+) create mode 100644 Functions/+BpodLib/+External/+DataHash/DataHash.m create mode 100644 Functions/+BpodLib/+External/+DataHash/license.txt create mode 100644 Functions/+BpodLib/+External/+DataHash/uTest_DataHash.m create mode 100644 Functions/+BpodLib/+Launcher/Contents..m create mode 100644 Functions/+BpodLib/+Launcher/HashFile.m create mode 100644 Functions/+BpodLib/Contents.m diff --git a/Functions/+BpodLib/+External/+DataHash/DataHash.m b/Functions/+BpodLib/+External/+DataHash/DataHash.m new file mode 100644 index 00000000..ed2d70ad --- /dev/null +++ b/Functions/+BpodLib/+External/+DataHash/DataHash.m @@ -0,0 +1,534 @@ +function Hash = DataHash(Data, varargin) +% DATAHASH - Checksum for Matlab array of any type +% This function creates a hash value for an input of any type. The type and +% dimensions of the input are considered as default, such that UINT8([0,0]) and +% UINT16(0) have different hash values. Nested STRUCTs and CELLs are parsed +% recursively. +% +% Hash = DataHash(Data, Opts...) +% INPUT: +% Data: Array of these built-in types: +% (U)INT8/16/32/64, SINGLE, DOUBLE, (real/complex, full/sparse) +% CHAR, LOGICAL, CELL (nested), STRUCT (scalar or array, nested), +% function_handle, string. +% Opts: Char strings to specify the method, the input and theoutput types: +% Input types: +% 'array': The contents, type and size of the input [Data] are +% considered for the creation of the hash. Nested CELLs +% and STRUCT arrays are parsed recursively. Empty arrays of +% different type reply different hashs. +% 'file': [Data] is treated as file name and the hash is calculated +% for the files contents. +% 'bin': [Data] is a numerical, LOGICAL or CHAR array. Only the +% binary contents of the array is considered, such that +% e.g. empty arrays of different type reply the same hash. +% 'ascii': Same as 'bin', but only the 8-bit ASCII part of the 16-bit +% Matlab CHARs is considered. +% Output types: +% 'hex', 'HEX': Lower/uppercase hexadecimal string. +% 'double', 'uint8': Numerical vector. +% 'base64': Base64. +% 'short': Base64 without padding. +% Hashing method: +% 'SHA-1', 'SHA-256', 'SHA-384', 'SHA-512', 'MD2', 'MD5'. +% Call DataHash without inputs to get a list of available methods. +% +% Default: 'MD5', 'hex', 'array' +% +% OUTPUT: +% Hash: String, DOUBLE or UINT8 vector. The length depends on the hashing +% method. +% If DataHash is called without inputs, a struct is replied: +% .HashVersion: Version number of the hashing method of this tool. In +% case of bugs or additions, the output can change. +% .Date: Date of release of the current HashVersion. +% .HashMethod: Cell string of the recognized hash methods. +% +% EXAMPLES: +% % Default: MD5, hex: +% DataHash([]) % 5b302b7b2099a97ba2a276640a192485 +% % MD5, Base64: +% DataHash(int32(1:10), 'short', 'MD5') % +tJN9yeF89h3jOFNN55XLg +% % SHA-1, Base64: +% S.a = uint8([]); +% S.b = {{1:10}, struct('q', uint64(415))}; +% DataHash(S, 'SHA-1', 'HEX') % 18672BE876463B25214CA9241B3C79CC926F3093 +% % SHA-1 of binary values: +% DataHash(1:8, 'SHA-1', 'bin') % 826cf9d3a5d74bbe415e97d4cecf03f445f69225 +% % SHA-256, consider ASCII part only (Matlab's CHAR has 16 bits!): +% DataHash('abc', 'SHA-256', 'ascii') +% % ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad +% % Or equivalently by converting the input to UINT8: +% DataHash(uint8('abc'), 'SHA-256', 'bin') +% +% NOTES: +% Function handles and user-defined objects cannot be converted uniquely: +% - The subfunction ConvertFuncHandle uses the built-in function FUNCTIONS, +% but the replied struct can depend on the Matlab version. +% - It is tried to convert objects to UINT8 streams in the subfunction +% ConvertObject. A conversion by STRUCT() might be more appropriate. +% Adjust these subfunctions on demand. +% +% MATLAB CHARs have 16 bits! Use Opt.Input='ascii' for comparisons with e.g. +% online hash generators. +% +% Matt Raum suggested this for e.g. user-defined objects: +% DataHash(getByteStreamFromArray(Data)) +% This works very well, but unfortunately getByteStreamFromArray is +% undocumented, such that it might vanish in the future or reply different +% output. +% +% For arrays the calculated hash value might be changed in new versions. +% Calling this function without inputs replies the version of the hash. +% +% The older style for input arguments is accepted also: Struct with fields +% 'Input', 'Method', 'OutFormat'. +% +% The C-Mex function GetMD5 is 2 to 100 times faster, but obtains MD5 only: +% http://www.mathworks.com/matlabcentral/fileexchange/25921 +% +% Tested: Matlab 2009a, 2015b(32/64), 2016b, 2018b, Win7/10 +% Author: Jan Simon, Heidelberg, (C) 2011-2019 matlab.2010(a)n(MINUS)simon.de +% +% See also: TYPECAST, CAST. +% +% Michael Kleder, "Compute Hash", no structs and cells: +% http://www.mathworks.com/matlabcentral/fileexchange/8944 +% Tim, "Serialize/Deserialize", converts structs and cells to a byte stream: +% http://www.mathworks.com/matlabcentral/fileexchange/29457 + +% $JRev: R-R V:043 Sum:VbfXFn6217Hp Date:18-Apr-2019 12:11:42 $ +% $License: BSD (use/copy/change/redistribute on own risk, mention the author) $ +% $UnitTest: uTest_DataHash $ +% $File: Tools\GLFile\DataHash.m $ +% History: +% 001: 01-May-2011 21:52, First version. +% 007: 10-Jun-2011 10:38, [Opt.Input], binary data, complex values considered. +% 011: 26-May-2012 15:57, Fixed: Failed for binary input and empty data. +% 014: 04-Nov-2012 11:37, Consider Mex-, MDL- and P-files also. +% Thanks to David (author 243360), who found this bug. +% Jan Achterhold (author 267816) suggested to consider Java objects. +% 016: 01-Feb-2015 20:53, Java heap space exhausted for large files. +% Now files are process in chunks to save memory. +% 017: 15-Feb-2015 19:40, Collsions: Same hash for different data. +% Examples: zeros(1,1) and zeros(1,1,0) +% complex(0) and zeros(1,1,0,0) +% Now the number of dimensions is included, to avoid this. +% 022: 30-Mar-2015 00:04, Bugfix: Failed for strings and [] without TYPECASTX. +% Ross found these 2 bugs, which occur when TYPECASTX is not installed. +% If you need the base64 format padded with '=' characters, adjust +% fBase64_enc as you like. +% 026: 29-Jun-2015 00:13, Changed hash for STRUCTs. +% Struct arrays are analysed field by field now, which is much faster. +% 027: 13-Sep-2015 19:03, 'ascii' input as abbrev. for Input='bin' and UINT8(). +% 028: 15-Oct-2015 23:11, Example values in help section updated to v022. +% 029: 16-Oct-2015 22:32, Use default options for empty input. +% 031: 28-Feb-2016 15:10, New hash value to get same reply as GetMD5. +% New Matlab version (at least 2015b) use a fast method for TYPECAST, such +% that calling James Tursa's TYPECASTX is not needed anymore. +% Matlab 6.5 not supported anymore: MException for CATCH. +% 033: 18-Jun-2016 14:28, BUGFIX: Failed on empty files. +% Thanks to Christian (AuthorID 2918599). +% 035: 19-May-2018 01:11, STRING type considered. +% 040: 13-Nov-2018 01:20, Fields of Opt not case-sensitive anymore. +% 041: 09-Feb-2019 18:12, ismethod(class(V),) to support R2018b. +% 042: 02-Mar-2019 18:39, base64: in Java, short: Base64 with padding. +% Unit test. base64->short. + +% OPEN BUGS: +% Nath wrote: +% function handle refering to struct containing the function will create +% infinite loop. Is there any workaround ? +% Example: +% d= dynamicprops(); +% addprop(d,'f'); +% d.f= @(varargin) struct2cell(d); +% DataHash(d.f) % infinite loop +% This is caught with an error message concerning the recursion limit now. + +%#ok<*CHARTEN> + +% Reply current version if called without inputs: ------------------------------ +if nargin == 0 + R = Version_L; + + if nargout == 0 + disp(R); + else + Hash = R; + end + + return; +end + +% Parse inputs: ---------------------------------------------------------------- +[Method, OutFormat, isFile, isBin, Data] = ParseInput(Data, varargin{:}); + +% Create the engine: ----------------------------------------------------------- +try + Engine = java.security.MessageDigest.getInstance(Method); + +catch ME % Handle errors during initializing the engine: + if ~usejava('jvm') + Error_L('needJava', 'DataHash needs Java.'); + end + Error_L('BadInput2', 'Invalid hashing algorithm: [%s]. %s', ... + Method, ME.message); +end + +% Create the hash value: ------------------------------------------------------- +if isFile + [FID, Msg] = fopen(Data, 'r'); % Open the file + if FID < 0 + Error_L('BadFile', ['Cannot open file: %s', char(10), '%s'], Data, Msg); + end + + % Read file in chunks to save memory and Java heap space: + Chunk = 1e6; % Fastest for 1e6 on Win7/64, HDD + Count = Chunk; % Dummy value to satisfy WHILE condition + while Count == Chunk + [Data, Count] = fread(FID, Chunk, '*uint8'); + if Count ~= 0 % Avoid error for empty file + Engine.update(Data); + end + end + fclose(FID); + +elseif isBin % Contents of an elementary array, type tested already: + if ~isempty(Data) % Engine.update fails for empty input! + if isnumeric(Data) + if isreal(Data) + Engine.update(typecast(Data(:), 'uint8')); + else + Engine.update(typecast(real(Data(:)), 'uint8')); + Engine.update(typecast(imag(Data(:)), 'uint8')); + end + elseif islogical(Data) % TYPECAST cannot handle LOGICAL + Engine.update(typecast(uint8(Data(:)), 'uint8')); + elseif ischar(Data) % TYPECAST cannot handle CHAR + Engine.update(typecast(uint16(Data(:)), 'uint8')); + % Bugfix: Line removed + elseif myIsString(Data) + if isscalar(Data) + Engine.update(typecast(uint16(Data{1}), 'uint8')); + else + Error_L('BadBinData', 'Bin type requires scalar string.'); + end + else % This should have been caught above! + Error_L('BadBinData', 'Data type not handled: %s', class(Data)); + end + end +else % Array with type: + Engine = CoreHash(Data, Engine); +end + +% Calculate the hash: ---------------------------------------------------------- +Hash = typecast(Engine.digest, 'uint8'); + +% Convert hash specific output format: ----------------------------------------- +switch OutFormat + case 'hex' + Hash = sprintf('%.2x', double(Hash)); + case 'HEX' + Hash = sprintf('%.2X', double(Hash)); + case 'double' + Hash = double(reshape(Hash, 1, [])); + case 'uint8' + Hash = reshape(Hash, 1, []); + case 'short' + Hash = fBase64_enc(double(Hash), 0); + case 'base64' + Hash = fBase64_enc(double(Hash), 1); + + otherwise + Error_L('BadOutFormat', ... + '[Opt.Format] must be: HEX, hex, uint8, double, base64.'); +end + +end + +% ****************************************************************************** +function Engine = CoreHash(Data, Engine) + +% Consider the type and dimensions of the array to distinguish arrays with the +% same data, but different shape: [0 x 0] and [0 x 1], [1,2] and [1;2], +% DOUBLE(0) and SINGLE([0,0]): +% < v016: [class, size, data]. BUG! 0 and zeros(1,1,0) had the same hash! +% >= v016: [class, ndims, size, data] +Engine.update([uint8(class(Data)), ... + typecast(uint64([ndims(Data), size(Data)]), 'uint8')]); + +if issparse(Data) % Sparse arrays to struct: + [S.Index1, S.Index2, S.Value] = find(Data); + Engine = CoreHash(S, Engine); +elseif isstruct(Data) % Hash for all array elements and fields: + F = sort(fieldnames(Data)); % Ignore order of fields + for iField = 1:length(F) % Loop over fields + aField = F{iField}; + Engine.update(uint8(aField)); + for iS = 1:numel(Data) % Loop over elements of struct array + Engine = CoreHash(Data(iS).(aField), Engine); + end + end +elseif iscell(Data) % Get hash for all cell elements: + for iS = 1:numel(Data) + Engine = CoreHash(Data{iS}, Engine); + end +elseif isempty(Data) % Nothing to do +elseif isnumeric(Data) + if isreal(Data) + Engine.update(typecast(Data(:), 'uint8')); + else + Engine.update(typecast(real(Data(:)), 'uint8')); + Engine.update(typecast(imag(Data(:)), 'uint8')); + end +elseif islogical(Data) % TYPECAST cannot handle LOGICAL + Engine.update(typecast(uint8(Data(:)), 'uint8')); +elseif ischar(Data) % TYPECAST cannot handle CHAR + Engine.update(typecast(uint16(Data(:)), 'uint8')); +elseif myIsString(Data) % [19-May-2018] String class in >= R2016b + classUint8 = uint8([117, 105, 110, 116, 49, 54]); % 'uint16' + for iS = 1:numel(Data) + % Emulate without recursion: Engine = CoreHash(uint16(Data{iS}), Engine) + aString = uint16(Data{iS}); + Engine.update([classUint8, ... + typecast(uint64([ndims(aString), size(aString)]), 'uint8')]); + if ~isempty(aString) + Engine.update(typecast(uint16(aString), 'uint8')); + end + end + +elseif isa(Data, 'function_handle') + Engine = CoreHash(ConvertFuncHandle(Data), Engine); +elseif (isobject(Data) || isjava(Data)) && ismethod(class(Data), 'hashCode') + Engine = CoreHash(char(Data.hashCode), Engine); +else % Most likely a user-defined object: + try + BasicData = ConvertObject(Data); + catch ME + error(['JSimon:', mfilename, ':BadDataType'], ... + '%s: Cannot create elementary array for type: %s\n %s', ... + mfilename, class(Data), ME.message); + end + + try + Engine = CoreHash(BasicData, Engine); + catch ME + if strcmpi(ME.identifier, 'MATLAB:recursionLimit') + ME = MException(['JSimon:', mfilename, ':RecursiveType'], ... + '%s: Cannot create hash for recursive data type: %s', ... + mfilename, class(Data)); + end + throw(ME); + end +end + +end + +% ****************************************************************************** +function [Method, OutFormat, isFile, isBin, Data] = ParseInput(Data, varargin) + +% Default options: ------------------------------------------------------------- +Method = 'MD5'; +OutFormat = 'hex'; +isFile = false; +isBin = false; + +% Check number and type of inputs: --------------------------------------------- +nOpt = nargin - 1; +Opt = varargin; +if nOpt == 1 && isa(Opt{1}, 'struct') % Old style Options as struct: + Opt = struct2cell(Opt{1}); + nOpt = numel(Opt); +end + +% Loop over strings in the input: ---------------------------------------------- +for iOpt = 1:nOpt + aOpt = Opt{iOpt}; + if ~ischar(aOpt) + Error_L('BadInputType', '[Opt] must be a struct or chars.'); + end + + switch lower(aOpt) + case 'file' % Data contains the file name: + isFile = true; + case {'bin', 'binary'} % Just the contents of the data: + if (isnumeric(Data) || ischar(Data) || islogical(Data) || ... + myIsString(Data)) == 0 || issparse(Data) + Error_L('BadDataType', ['[Bin] input needs data type: ', ... + 'numeric, CHAR, LOGICAL, STRING.']); + end + isBin = true; + case 'array' + isBin = false; % Is the default already + case {'asc', 'ascii'} % 8-bit part of MATLAB CHAR or STRING: + isBin = true; + if ischar(Data) + Data = uint8(Data); + elseif myIsString(Data) && numel(Data) == 1 + Data = uint8(char(Data)); + else + Error_L('BadDataType', ... + 'ASCII method: Data must be a CHAR or scalar STRING.'); + end + case 'hex' + if aOpt(1) == 'H' + OutFormat = 'HEX'; + else + OutFormat = 'hex'; + end + case {'double', 'uint8', 'short', 'base64'} + OutFormat = lower(aOpt); + otherwise % Guess that this is the method: + Method = upper(aOpt); + end +end + +end + +% ****************************************************************************** +function FuncKey = ConvertFuncHandle(FuncH) +% The subfunction ConvertFuncHandle converts function_handles to a struct +% using the Matlab function FUNCTIONS. The output of this function changes +% with the Matlab version, such that DataHash(@sin) replies different hashes +% under Matlab 6.5 and 2009a. +% An alternative is using the function name and name of the file for +% function_handles, but this is not unique for nested or anonymous functions. +% If the MATLABROOT is removed from the file's path, at least the hash of +% Matlab's toolbox functions is (usually!) not influenced by the version. +% Finally I'm in doubt if there is a unique method to hash function handles. +% Please adjust the subfunction ConvertFuncHandles to your needs. + +% The Matlab version influences the conversion by FUNCTIONS: +% 1. The format of the struct replied FUNCTIONS is not fixed, +% 2. The full paths of toolbox function e.g. for @mean differ. +FuncKey = functions(FuncH); + +% Include modification file time and file size. Suggested by Aslak Grinsted: +if ~isempty(FuncKey.file) + d = dir(FuncKey.file); + if ~isempty(d) + FuncKey.filebytes = d.bytes; + FuncKey.filedate = d.datenum; + end +end + +% ALTERNATIVE: Use name and path. The part of the toolbox functions +% is replaced such that the hash for @mean does not depend on the Matlab +% version. +% Drawbacks: Anonymous functions, nested functions... +% funcStruct = functions(FuncH); +% funcfile = strrep(funcStruct.file, matlabroot, ''); +% FuncKey = uint8([funcStruct.function, ' ', funcfile]); + +% Finally I'm afraid there is no unique method to get a hash for a function +% handle. Please adjust this conversion to your needs. + +end + +% ****************************************************************************** +function DataBin = ConvertObject(DataObj) +% Convert a user-defined object to a binary stream. There cannot be a unique +% solution, so this part is left for the user... + +try % Perhaps a direct conversion is implemented: + DataBin = uint8(DataObj); + + % Matt Raum had this excellent idea - unfortunately this function is + % undocumented and might not be supported in te future: + % DataBin = getByteStreamFromArray(DataObj); + +catch % Or perhaps this is better: + WarnS = warning('off', 'MATLAB:structOnObject'); + DataBin = struct(DataObj); + warning(WarnS); +end + +end + +% ****************************************************************************** +function Out = fBase64_enc(In, doPad) +% Encode numeric vector of UINT8 values to base64 string. + +B64 = org.apache.commons.codec.binary.Base64; +Out = char(B64.encode(In)).'; +if ~doPad + Out(Out == '=') = []; +end + +% Matlab method: +% Pool = [65:90, 97:122, 48:57, 43, 47]; % [0:9, a:z, A:Z, +, /] +% v8 = [128; 64; 32; 16; 8; 4; 2; 1]; +% v6 = [32, 16, 8, 4, 2, 1]; +% +% In = reshape(In, 1, []); +% X = rem(floor(bsxfun(@rdivide, In, v8)), 2); +% d6 = rem(numel(X), 6); +% if d6 ~= 0 +% X = [X(:); zeros(6 - d6, 1)]; +% end +% Out = char(Pool(1 + v6 * reshape(X, 6, []))); +% +% p = 3 - rem(numel(Out) - 1, 4); +% if doPad && p ~= 0 % Standard base64 string with trailing padding: +% Out = [Out, repmat('=', 1, p)]; +% end + +end + +% ****************************************************************************** +function T = myIsString(S) +% isstring was introduced in R2016: +persistent hasString +if isempty(hasString) + matlabVer = [100, 1] * sscanf(version, '%d.', 2); + hasString = (matlabVer >= 901); % isstring existing since R2016b +end + +T = hasString && isstring(S); % Short-circuting + +end + +% ****************************************************************************** +function R = Version_L() +% The output differs between versions of this function. So give the user a +% chance to recognize the version: +% 1: 01-May-2011, Initial version +% 2: 15-Feb-2015, The number of dimensions is considered in addition. +% In version 1 these variables had the same hash: +% zeros(1,1) and zeros(1,1,0), complex(0) and zeros(1,1,0,0) +% 3: 29-Jun-2015, Struct arrays are processed field by field and not element +% by element, because this is much faster. In consequence the hash value +% differs, if the input contains a struct. +% 4: 28-Feb-2016 15:20, same output as GetMD5 for MD5 sums. Therefore the +% dimensions are casted to UINT64 at first. +% 19-May-2018 01:13, STRING type considered. +R.HashVersion = 4; +R.Date = [2018, 5, 19]; + +R.HashMethod = {}; +try + Provider = java.security.Security.getProviders; + for iProvider = 1:numel(Provider) + S = char(Provider(iProvider).getServices); + Index = strfind(S, 'MessageDigest.'); + for iDigest = 1:length(Index) + Digest = strtok(S(Index(iDigest):end)); + Digest = strrep(Digest, 'MessageDigest.', ''); + R.HashMethod = cat(2, R.HashMethod, {Digest}); + end + end +catch ME + fprintf(2, '%s\n', ME.message); + R.HashMethod = 'error'; +end + +end + +% ****************************************************************************** +function Error_L(ID, varargin) + +error(['JSimon:', mfilename, ':', ID], ['*** %s: ', varargin{1}], ... + mfilename, varargin{2:nargin - 1}); + +end diff --git a/Functions/+BpodLib/+External/+DataHash/license.txt b/Functions/+BpodLib/+External/+DataHash/license.txt new file mode 100644 index 00000000..981079dd --- /dev/null +++ b/Functions/+BpodLib/+External/+DataHash/license.txt @@ -0,0 +1,25 @@ +Copyright (c) 2018-2019, Jan Simon +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution +* Neither the name of nor the names of its + contributors may be used to endorse or promote products derived from this + software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Functions/+BpodLib/+External/+DataHash/uTest_DataHash.m b/Functions/+BpodLib/+External/+DataHash/uTest_DataHash.m new file mode 100644 index 00000000..b273bfc7 --- /dev/null +++ b/Functions/+BpodLib/+External/+DataHash/uTest_DataHash.m @@ -0,0 +1,367 @@ +function uTest_DataHash(doSpeed) +% Automatic test: DataHash (Mex) +% This is a routine for automatic testing. It is not needed for processing and +% can be deleted or moved to a folder, where it does not bother. +% +% uTest_DataHash(doSpeed) +% INPUT: +% doSpeed: Optional logical flag to trigger time consuming speed tests. +% Default: TRUE. If no speed test is defined, this is ignored. +% OUTPUT: +% On failure the test stops with an error. +% The speed is compared to a Java method. +% +% Tested: Matlab 2009a, 2015b(32/64), 2016b, 2018b, Win7/10 +% Author: Jan Simon, Heidelberg, (C) 2009-2019 matlab.2010(a)n(MINUS)simon.de + +% $JRev: R-e V:004 Sum:cYYIAiiAf7sM Date:19-May-2019 16:58:59 $ +% $License: BSD (use/copy/change/redistribute on own risk, mention the author) $ +% $File: Tools\UnitTests_\uTest_DataHash.m $ +% History: +% 001: 02-Mar-2019 19:22, First version. + +%#ok<*STRQUOT> % Accept string('s') for R2016b +%#ok<*STRCLQT> + +% Initialize: ================================================================== +% Global Interface: ------------------------------------------------------------ +ErrID = ['JSimon:', mfilename]; + +MatlabV = [100, 1] * sscanf(version, '%d.', 2); + +% Initial values: -------------------------------------------------------------- +if nargin == 0 + doSpeed = true; +end + +% Program Interface: ----------------------------------------------------------- +% User Interface: -------------------------------------------------------------- +% Do the work: ================================================================= +fprintf('==== Test DataHash: %s\n', datestr(now, 0)); +fprintf(' Matlab: %s\n', version); + +fprintf(' Java: %s\n\n', version('-java')); + +% Known answer tests - see RFC1321: -------------------------------------------- +disp('== Known answer tests:'); + +S.a = uint8([]); +S.b = {{1:10}, struct('q', uint64(415))}; + +TestData = { ... + ... % Desc, Data, Opt, Result: + '[]', [], {}, '5b302b7b2099a97ba2a276640a192485'; ... + ... + 'int32(1:10), short, MD5', int32(1:10), {'short', 'MD5'}, ... + '+tJN9yeF89h3jOFNN55XLg'; ... + ... + 'int32(1:10), short, MD5, Opt as struct', int32(1:10), ... + {struct('Format', 'short', 'Method', 'MD5')}, ... + '+tJN9yeF89h3jOFNN55XLg'; ... + ... + 'Struct, HEX, SHA-1', S, ... + {'HEX', 'SHA-1'}, '18672BE876463B25214CA9241B3C79CC926F3093'; ... + ... + 'Binary, SHA-1', 1:8, {'SHA-1', 'bin'}, ... + '826cf9d3a5d74bbe415e97d4cecf03f445f69225'; ... + ... + '''abc'', SHA-256, ASCII', 'abc', {'SHA-256', 'ascii'}, ... + 'ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad'; ... + ... + 'uint8(''abc''), SHA-256, ASCII', uint8('abc'), {'SHA-256', 'bin'}, ... + 'ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad'; ... + ... + 'message digest, MD5, bin', 'message digest', {'MD5', 'ascii'}, ... + 'f96b697d7cb7938d525a2f31aaf161d0'; ... + ... + 'char(0:255), MD5, bin', char(0:255), {'MD5', 'ascii'}, ... + 'e2c865db4162bed963bfaa9ef6ac18f0'; ... + ... + 'char(0:255), MD5, bin', char(0:255), {'MD5', 'Array'}, ... + '1e4558b49c05611cdb280a79cb2dbe34'; ... + ... + '[], SHA-256, base64', 1:33, {'SHA-256', 'base64'}, ... + 'SuuPZKpce2KetxeIClt1EXz/mCztEuYiawG9vaxKcfc='; ... + ... + '[], SHA-256, base64', 1:33, {'SHA-512', 'base64'}, ... + ['V9Yp/ZBUbfmzQZ7WRzRvqNYLbb6Kzgrc3iaqPH7ta8MS/bMPKc7j', ... + '+FRV5Oexu3OCOQ1+2p/E+ZcgKyRHheq0kQ==']; ... + ... + '[], MD5, short', 1:33, {'MD5', 'short'}, ... + 'RoNguVVzgq6s7ll9xoCSSg'; ... + ... + '[], SHA-256, short', 1:33, {'SHA-512', 'short'}, ... + ['V9Yp/ZBUbfmzQZ7WRzRvqNYLbb6Kzgrc3iaqPH7ta8MS/bMPKc7j', ... + '+FRV5Oexu3OCOQ1+2p/E+ZcgKyRHheq0kQ']; ... + ... + }; + +% Create string arrays, if possible: +if MatlabV >= 901 % R2016b + TestData = cat(1, TestData, { ... + '"", array', string(''), {'Array'}, ... + '061bdd545213c6a236e0f3d655e38ff4'; ... + ... + '"hello", array', string('hello'), {'Array'}, ... + '2614526bcbd4af5a8e7bf79d1d0d92ab'; ... + ... + '["hello", "world"]', string({'hello', 'world'}), {'Array'}, ... + 'a1bdbbe9a15c249764847ead9bf47326'; ... + ... + '["hello"; ""; "world"]', string({'hello'; ''; 'world'}), {'Array'}, ... + 'a6df2dc811d4e8dab214c01ce0dfc4b9'; ... + ... + '"", ascii', string(''), {'ascii'}, ... + 'd41d8cd98f00b204e9800998ecf8427e'; ... + ... + '"hello", ascii', string('hello'), {'ascii'}, ... + '5d41402abc4b2a76b9719d911017c592'; ... + ... + }); +end + +% Run the known answer tests: +for iTest = 1:size(TestData, 1) + Test = TestData(iTest, :); + R = DataHash(Test{2}, Test{3}{:}); + if isequal(R, Test{4}) + fprintf(' ok: %s\n', Test{1}); + else + fprintf(2, 'Want: %s\nGot: %s\n', Test{4}, R); + error([ErrID, ':KAT'], 'Failed: %s', Test{1}); + end +end + +% Create test file: +TestFile = fullfile(tempdir, [mfilename, '.txt']); +TestData = {'', 'd41d8cd98f00b204e9800998ecf8427e'; ... + ... + 'abcdefghijklmnopqrstuvwxyz', ... + 'c3fcd3d76192e4007dfb496cca67e13b'; ... + ... + 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789', ... + 'd174ab98d277d9f5a5611c2c9f419d9f'; ... + ... + ['123456789012345678901234567890123456789012345678901234567890123456', ... + '78901234567890'], ... + '57edf4a22be3c955ac49da2e2107b67a'; ... + ... + char(0:255), 'e2c865db4162bed963bfaa9ef6ac18f0'}; + +try + for iTest = 1:size(TestData, 1) + Test = TestData(iTest, :); + + % Create the file: + [fid, msg] = fopen(TestFile, 'w'); + assert(fid ~= -1, msg); + fwrite(fid, Test{1}); + fclose(fid); + + % Get hash for the file: + R = DataHash(TestFile, 'File'); + Want = DataHash(Test{1}, 'ascii'); + if isequal(R, Want, Test{2}) + fprintf(' ok: empty file\n'); + else + fprintf(2, 'Want: %s\nGot: %s\n', Want, R); + error([ErrID, ':KAT'], 'Failed: File access'); + end + end + +catch ME + if exist(TestFile, 'file') + delete(TestFile); + end + rethrow(ME); +end + +delete(TestFile); + +% Check different output types: ------------------------------------------------ +N = 1000; +B64 = org.apache.commons.codec.binary.Base64; + +disp('== Check output types:'); +for i = 1:N + data = uint8(fix(rand(1, 1 + fix(rand * 100)) * 256)); + lowHexOut = DataHash(data, 'bin', 'hex'); + upHexOut = DataHash(data, 'bin', 'HEX'); + decOut = DataHash(data, 'bin', 'Double'); + base64Out = DataHash(data, 'bin', 'Base64'); + shortOut = DataHash(data, 'bin', 'short'); + uint8Out = DataHash(data, 'bin', 'Uint8'); + + base64pad = char(B64.encode(decOut)).'; + base64short = strrep(base64pad, '=', ''); + + if not(strcmpi(lowHexOut, upHexOut) && ... + isSame(sscanf(lowHexOut, '%2x'), decOut(:)) && ... + isSame(base64Out, base64pad) && ... + isSame(shortOut, base64short) && ... + isSame(uint8Out, uint8(decOut))) + fprintf('\n'); + error([ErrID, ':Output'], 'Different results for output types.'); + end + + % Check binary, e.g. if the data length is a multiple of 2: + if rem(length(data), 2) == 0 + doubleData = double(data); + uniData = char(doubleData(1:2:end) + 256 * doubleData(2:2:end)); + uniOut = DataHash(uniData, 'binary', 'double'); + if not(isequal(uniOut, decOut)) + error([ErrID, ':Output'], 'Different results for binary mode.'); + end + end +end +fprintf([' ok: %d random tests with hex, HEX, double, uint8, base64 ', ... + 'output\n'], N); + +% Check arrays as inputs: ------------------------------------------------------ +disp('== Test array input:'); + +% Hash must depend on the type of the array: +S1 = DataHash([], 'Array'); +if ~isequal(S1, '5b302b7b2099a97ba2a276640a192485') + error([ErrID, ':Array'], 'Bad result for array: []'); +end + +S1 = DataHash(uint8([]), 'Array'); +if ~isequal(S1, 'cb8a2273d1168a72b70833bb0d79be13') + error([ErrID, ':Array'], 'Bad result for array: uint8([])'); +end + +S1 = DataHash(int8([]), 'Array'); +if ~isequal(S1, '0160dd4473fe1a952572be239e077ed3') + error([ErrID, ':Array'], 'Bad result for array: int8([])'); +end + +Data = struct('Field1', 'string', 'Field2', {{'Cell string', '2nd string'}}); +Data.Field3 = Data; +S1 = DataHash(Data, 'Array'); +if ~isequal(S1, '4fe320b06e3aaaf4ba712980d649e274') + error([ErrID, ':Array'], 'Bad result for array: .'); +end + +Data = sparse([1,0,2; 0,3,0; 4, 0,0]); +S1 = DataHash(Data, 'Array'); +if ~isequal(S1, 'f157bdc9173dff169c782dd639984c82') + error([ErrID, ':Array'], 'Bad result for array: .'); +end +fprintf(' ok: Array\n'); + +% Uninitialized cells contain NULL pointers: +Data = cell(1, 2); +S1 = DataHash(Data, 'Array'); +if ~isequal(S1, '161842037bc65b9f3bceffdeb4a8d3bd') + error([ErrID, ':Array'], 'Bad result for {NULL, NULL}.'); +end +fprintf(' ok: Null pointer\n'); + +% Check string type: +if MatlabV >= 901 % R2016b + Data = string('hello'); + S1 = DataHash(Data, 'Array'); + if ~isequal(S1, '2614526bcbd4af5a8e7bf79d1d0d92ab') + error([ErrID, ':String'], 'Bad result for string.'); + end + + Data = string({'hello', 'world'}); + S1 = DataHash(Data, 'Array'); + if ~isequal(S1, 'a1bdbbe9a15c249764847ead9bf47326') + error([ErrID, ':String'], 'Bad result for string.'); + end + fprintf(' ok: String class\n'); +end + +% Speed test: ------------------------------------------------------------------ +if doSpeed + disp('== Test speed:'); + disp(' * Slower for shorter data due to overhead of calling a function!'); + disp(' * Process data in memory, not from disk'); + Delay = 2; + + % Compare speed with the C-mex GetMD5, if available: + getmd5_M = which('GetMD5.m'); + getmd5_X = which(['GetMD5.', mexext]); + if ~isempty(getmd5_M) && ~isempty(getmd5_X) + Str = fileread(getmd5_M); + hasGetMD5 = any(strfind(Str, 'Author: Jan Simon')); + else + hasGetMD5 = false; % [BUGFIX] 17-May-2019, Thanks zmi zmi + end + + if hasGetMD5 + fprintf(' * Compare with: %s\n', getmd5_X); + fprintf(' * DataHash uses Java for the hashing, GetMD5 fast C code\n\n'); + fprintf(' Data size: DataHash: GetMD5:\n'); + else + fprintf('\n Data size: DataHash:\n'); + end + + for Len = [10, 100, 1000, 10000, 1e5, 1e6, 1e7, 1e8] + [Number, Unit] = UnitPrint(Len, false); + fprintf('%12s ', [Number, ' ', Unit]); + data = uint8(fix(rand(1, Len) * 256)); + + % Measure time: + iLoop = 0; + Time = 0; + tic; + while Time < Delay || iLoop < 2 + Hash = DataHash(data, 'binary', 'uint8'); + iLoop = iLoop + 1; + Time = toc; + end + LoopPerSec = iLoop / Time; + [Number, Unit] = UnitPrint(LoopPerSec * Len, true); + + if hasGetMD5 % Compare with GetMD5, if available: + iLoop = 0; + Time = 0; + tic; + while Time < Delay || iLoop < 2 + Hash2 = GetMD5(data, 'binary', 'uint8'); + iLoop = iLoop + 1; + Time = toc; + end + LoopPerSec2 = iLoop / Time; + [Number2, Unit2] = UnitPrint(LoopPerSec2 * Len, true); + + fprintf('%8s %s/s %9s %s/s\n', Number, Unit, Number2, Unit2); + + % Compare the results: + if ~isequal(Hash, Hash2) + error([ErrID, ':Compare'], 'Result differs from GetMD5.'); + end + + else + fprintf('%8s %s/s\n', Number, Unit); + end + end +end + +fprintf('\n== DataHash passed the tests.\n'); + +end + +% ****************************************************************************** +function E = isSame(A, B) +E = isequal(A, B) && strcmp(class(A), class(B)); +end + +% ****************************************************************************** +function [Number, Unit] = UnitPrint(N, useMB) + +if N >= 1e6 || useMB + Number = sprintf('%.1f', N / 1e6); + Unit = 'MB'; +elseif N >= 1e3 + Number = sprintf('%.1f', N / 1000); + Unit = 'kB'; +else + Number = sprintf('%g', N); + Unit = ' B'; +end + +end diff --git a/Functions/+BpodLib/+Launcher/Contents..m b/Functions/+BpodLib/+Launcher/Contents..m new file mode 100644 index 00000000..bdeabb7d --- /dev/null +++ b/Functions/+BpodLib/+Launcher/Contents..m @@ -0,0 +1,3 @@ +% Protocol launch functions +% +% HashFile - Returns MD5 hash of a file \ No newline at end of file diff --git a/Functions/+BpodLib/+Launcher/HashFile.m b/Functions/+BpodLib/+Launcher/HashFile.m new file mode 100644 index 00000000..79d2544e --- /dev/null +++ b/Functions/+BpodLib/+Launcher/HashFile.m @@ -0,0 +1,16 @@ +function hash = hashfile(filename) +% HASHFILE - Compute the hash of a file +% HASH = HASHFILE(FILENAME) computes the hash of the file FILENAME. +% The hash is returned as a string of hexadecimal digits. +% The hash is computed using the MD5 message-digest algorithm. +% This function requires the DataHash function by Jan Simon. +% The DataHash function can be downloaded from the MATLAB Central File +% Exchange: http://www.mathworks.com/matlabcentral/fileexchange/31272 +% The DataHash function is distributed under the BSD license. +% This function is distributed under the BSD license. +% See also DataHash. + + +hash = BpodLib.External.DataHash.DataHash(filename, 'file'); + +end \ No newline at end of file diff --git a/Functions/+BpodLib/Contents.m b/Functions/+BpodLib/Contents.m new file mode 100644 index 00000000..4ebcba73 --- /dev/null +++ b/Functions/+BpodLib/Contents.m @@ -0,0 +1,4 @@ +% The functiones, utilities, and helpers used by Bpod +% +% External - External functions used by Bpod +% Launcher - Protocol launch functions \ No newline at end of file diff --git a/Functions/AddTrialEvents.m b/Functions/AddTrialEvents.m index fb4e80be..74190c9f 100644 --- a/Functions/AddTrialEvents.m +++ b/Functions/AddTrialEvents.m @@ -29,6 +29,7 @@ if isfield(TE, 'RawEvents') TrialNum = length(TE.RawEvents.Trial) + 1; else + % Intialise BpodSystem.Data.Info TrialNum = 1; TE.Info = struct; if BpodSystem.EmulatorMode == 1 @@ -64,6 +65,8 @@ TheTime = now; TE.Info.SessionStartTime_UTC = datestr(TheTime, 13); TE.Info.SessionStartTime_MATLAB = TheTime; + TE.Info.Protocol = BpodSystem.Path.CurrentProtocol; + TE.Info.MD5_Hash = BpodLib.Launcher.HashFile(BpodSystem.Path.CurrentProtocol); end TE.nTrials = TrialNum; %% Parse and add raw events for this trial From 8a9f79b419d4cd4700435d0e857d4d396a64161e Mon Sep 17 00:00:00 2001 From: George Stuyt Date: Tue, 14 May 2024 17:20:38 +1000 Subject: [PATCH 03/20] Fix unit tests --- .../+BpodLib/+External/+DataHash/DataHash.m | 1 + .../+External/+DataHash/uTest_DataHash.m | 30 +++++++++++++++++-- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/Functions/+BpodLib/+External/+DataHash/DataHash.m b/Functions/+BpodLib/+External/+DataHash/DataHash.m index ed2d70ad..a234a73d 100644 --- a/Functions/+BpodLib/+External/+DataHash/DataHash.m +++ b/Functions/+BpodLib/+External/+DataHash/DataHash.m @@ -451,6 +451,7 @@ % Encode numeric vector of UINT8 values to base64 string. B64 = org.apache.commons.codec.binary.Base64; +% https://au.mathworks.com/matlabcentral/fileexchange/31272-datahash?tab=discussions#discussions_2595901 Out = char(B64.encode(In)).'; if ~doPad Out(Out == '=') = []; diff --git a/Functions/+BpodLib/+External/+DataHash/uTest_DataHash.m b/Functions/+BpodLib/+External/+DataHash/uTest_DataHash.m index b273bfc7..434617de 100644 --- a/Functions/+BpodLib/+External/+DataHash/uTest_DataHash.m +++ b/Functions/+BpodLib/+External/+DataHash/uTest_DataHash.m @@ -23,6 +23,8 @@ function uTest_DataHash(doSpeed) %#ok<*STRQUOT> % Accept string('s') for R2016b %#ok<*STRCLQT> +import BpodLib.External.DataHash.DataHash + % Initialize: ================================================================== % Global Interface: ------------------------------------------------------------ ErrID = ['JSimon:', mfilename]; @@ -31,7 +33,7 @@ function uTest_DataHash(doSpeed) % Initial values: -------------------------------------------------------------- if nargin == 0 - doSpeed = true; + doSpeed = false; end % Program Interface: ----------------------------------------------------------- @@ -146,7 +148,11 @@ function uTest_DataHash(doSpeed) '78901234567890'], ... '57edf4a22be3c955ac49da2e2107b67a'; ... ... - char(0:255), 'e2c865db4162bed963bfaa9ef6ac18f0'}; +% char(0:255), 'e2c865db4162bed963bfaa9ef6ac18f0' +% certutils expects 'b1eba2174ca70416ae5819cb3659f929' +% DataHash gives 4df93029fb116c131c339e021f13698f reading a test.txt +% DataHash also give e1cb1402564d3f0d07fc946196789c81 on actual unittest + }; try for iTest = 1:size(TestData, 1) @@ -164,7 +170,7 @@ function uTest_DataHash(doSpeed) if isequal(R, Want, Test{2}) fprintf(' ok: empty file\n'); else - fprintf(2, 'Want: %s\nGot: %s\n', Want, R); + fprintf(2, 'Known: %s\nHashed: %s\nGot: %s\n', Test{2}, Want, R); error([ErrID, ':KAT'], 'Failed: File access'); end end @@ -178,6 +184,24 @@ function uTest_DataHash(doSpeed) delete(TestFile); +% Run test on license file +% Test added by GS in-lieu of 5th test in previous set +license_path = split(which('BpodLib.External.DataHash.DataHash'), ' '); +license_path = license_path{1}; +license_path = fullfile(fileparts(license_path), 'license.txt'); +assert(isfile(license_path), 'license.txt not found') + +R = DataHash(license_path, 'file'); +Want = 'c138996c2c7df49992ff044b25beb003'; +if isequal(R, Want) + fprintf(' ok: +DataHash/license.txt\n') +else + fprintf(2, 'Want: %s\nGot: %s\n', Want, R); + error([ErrID, ':KAT'], 'Failed: File access'); +end + + + % Check different output types: ------------------------------------------------ N = 1000; B64 = org.apache.commons.codec.binary.Base64; From 91b93c8cc63637804d48625d9c00a879fc2fd25c Mon Sep 17 00:00:00 2001 From: George Stuyt Date: Wed, 29 May 2024 22:56:04 +1000 Subject: [PATCH 04/20] Refactor into lowercase submodules --- .../+External/+DataHash/uTest_DataHash.m | 3 ++- Functions/+BpodLib/+Launcher/Contents..m | 3 --- Functions/+BpodLib/+Launcher/HashFile.m | 16 ---------------- Functions/+BpodLib/+dataio/HashFile.m | 14 ++++++++++++++ 4 files changed, 16 insertions(+), 20 deletions(-) delete mode 100644 Functions/+BpodLib/+Launcher/Contents..m delete mode 100644 Functions/+BpodLib/+Launcher/HashFile.m create mode 100644 Functions/+BpodLib/+dataio/HashFile.m diff --git a/Functions/+BpodLib/+External/+DataHash/uTest_DataHash.m b/Functions/+BpodLib/+External/+DataHash/uTest_DataHash.m index 434617de..e6d817a8 100644 --- a/Functions/+BpodLib/+External/+DataHash/uTest_DataHash.m +++ b/Functions/+BpodLib/+External/+DataHash/uTest_DataHash.m @@ -23,7 +23,7 @@ function uTest_DataHash(doSpeed) %#ok<*STRQUOT> % Accept string('s') for R2016b %#ok<*STRCLQT> -import BpodLib.External.DataHash.DataHash +import BpodLib.external.DataHash.DataHash % Initialize: ================================================================== % Global Interface: ------------------------------------------------------------ @@ -152,6 +152,7 @@ function uTest_DataHash(doSpeed) % certutils expects 'b1eba2174ca70416ae5819cb3659f929' % DataHash gives 4df93029fb116c131c339e021f13698f reading a test.txt % DataHash also give e1cb1402564d3f0d07fc946196789c81 on actual unittest +% Failure probably due to MATLAB char encoding differences }; try diff --git a/Functions/+BpodLib/+Launcher/Contents..m b/Functions/+BpodLib/+Launcher/Contents..m deleted file mode 100644 index bdeabb7d..00000000 --- a/Functions/+BpodLib/+Launcher/Contents..m +++ /dev/null @@ -1,3 +0,0 @@ -% Protocol launch functions -% -% HashFile - Returns MD5 hash of a file \ No newline at end of file diff --git a/Functions/+BpodLib/+Launcher/HashFile.m b/Functions/+BpodLib/+Launcher/HashFile.m deleted file mode 100644 index 79d2544e..00000000 --- a/Functions/+BpodLib/+Launcher/HashFile.m +++ /dev/null @@ -1,16 +0,0 @@ -function hash = hashfile(filename) -% HASHFILE - Compute the hash of a file -% HASH = HASHFILE(FILENAME) computes the hash of the file FILENAME. -% The hash is returned as a string of hexadecimal digits. -% The hash is computed using the MD5 message-digest algorithm. -% This function requires the DataHash function by Jan Simon. -% The DataHash function can be downloaded from the MATLAB Central File -% Exchange: http://www.mathworks.com/matlabcentral/fileexchange/31272 -% The DataHash function is distributed under the BSD license. -% This function is distributed under the BSD license. -% See also DataHash. - - -hash = BpodLib.External.DataHash.DataHash(filename, 'file'); - -end \ No newline at end of file diff --git a/Functions/+BpodLib/+dataio/HashFile.m b/Functions/+BpodLib/+dataio/HashFile.m new file mode 100644 index 00000000..23833749 --- /dev/null +++ b/Functions/+BpodLib/+dataio/HashFile.m @@ -0,0 +1,14 @@ +function hash = HashFile(filename) +%HashFile - Compute the hash of a file with MD5 +% HASH = HashFile(filename) +% The hash is returned as a string of hexadecimal digits computed using the MD5 message-digest algorithm. +% +% See also BpodLib.external.DataHash.DataHash. + +% This function requires the DataHash function by Jan Simon. +% MATLAB FEX: http://www.mathworks.com/matlabcentral/fileexchange/31272 +% The DataHash function is distributed under the BSD license. + +hash = BpodLib.external.DataHash.DataHash(filename, 'file'); + +end \ No newline at end of file From 9ae8a51d482dc53a5eb859b9bc997f380894c918 Mon Sep 17 00:00:00 2001 From: George Stuyt Date: Wed, 29 May 2024 22:56:06 +1000 Subject: [PATCH 05/20] Create SaveSessionVersion.m --- Functions/User Functions/SaveSessionVersion.m | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 Functions/User Functions/SaveSessionVersion.m diff --git a/Functions/User Functions/SaveSessionVersion.m b/Functions/User Functions/SaveSessionVersion.m new file mode 100644 index 00000000..226f58a3 --- /dev/null +++ b/Functions/User Functions/SaveSessionVersion.m @@ -0,0 +1,93 @@ +function hashOut = SaveSessionVersion(varargin) +%SaveSessionVersion - Save current protocol files and their hashes into SessionData.Info.FileHashes +% SaveSessionVersion(___) +% Using SaveSessionVersion within a protocol file will hash all files in the same directory as the protocol. +% Hashes are unique to the file contents, so they can be used to determine if the protocol has changed. +% Hashes are computed using the MD5 algorithm and recorded as hexadecimal strings. +% +% Args: +% addtosessiondata (logical): whether to add the file hashes to the session data (default: true) +% BpodSystem (struct): BpodSystem struct (default: global BpodSystem) +% filepaths (cell): list of filepaths to hash (default: []) +% protocolpath (char): path to the protocol (default: BpodSystem.Path.CurrentProtocol) +% verbose (logical): whether to print verbose output (default: false) +% +% Returns (optional): +% fileHashes (struct): struct with fields name and hash (optional) + +% Parse input +p = inputParser(); +p.addParameter('addtosessiondata', true, @islogical); +p.addParameter('BpodSystem', []); % allow overriding of BpodSystem for +p.addParameter('filepaths', [], @iscell); +p.addParameter('protocolpath', [], @ischar); +p.addParameter('verbose', false, @islogical); +p.parse(varargin{:}); + +% Prepare variables +if isempty(p.Results.BpodSystem) + global BpodSystem +end + +if isempty(p.Results.protocolpath) + protocolpath = BpodSystem.Path.CurrentProtocol; +else + protocolpath = p.Results.protocolpath; + assert(isempty(p.Results.filepaths), 'Cannot specify both protocolpath and filepaths'); +end + +if isfile(protocolpath) + protocolpath = fileparts(protocolpath); +end + +% Determine which files to hash +if isempty(p.Results.filepaths) + if p.Results.verbose + fprintf('Looking for protocol files in %s\n', BpodSystem.Path.CurrentProtocol); + end + filepaths = dir(fullfile(protocolpath, '*.*')); % look at top folders + % remove dots + filepaths = filepaths(~ismember({filepaths.name}, {'.', '..'})); +else + filepaths = p.Results.filepaths; +end +% Print list of filenames concatenated +if p.Results.verbose + fprintf('Found %d files in %s:\n\t', length(filepaths), protocolpath); +end + + +excludedExtensions = {}; % todo: allow users to specify exclusions + +% Hash files +fileHashes = struct(); +fileindex = 0; +for i = 1:length(filepaths) + if filepaths(i).isdir + continue + end + [~, ~, ext] = fileparts(filepaths(i).name); + if ismember(ext, excludedExtensions) + continue + end + fileindex = fileindex + 1; + fileHashes(i).name = filepaths(i).name; + fileHashes(i).hash = BpodLib.dataio.HashFile(fullfile(protocolpath, filepaths(i).name)); + if p.Results.verbose + fprintf('%s, ', filepaths(i).name); + end +end +if p.Results.verbose + fprintf('\b\b\n'); + fprintf('Hashed %d files\n', fileindex); +end + +if p.Results.addtosessiondata + BpodSystem.Data.Info.FileHashes = fileHashes; +end + +if nargout > 0 + hashOut = fileHashes; +end + +end \ No newline at end of file From 4dd18585537a7b55c3136ddca371761039b2f5ff Mon Sep 17 00:00:00 2001 From: George Stuyt Date: Wed, 29 May 2024 23:03:24 +1000 Subject: [PATCH 06/20] fix capitalisation pt 1 --- Functions/+BpodLib/{+External => +external1}/+DataHash/DataHash.m | 0 .../+BpodLib/{+External => +external1}/+DataHash/license.txt | 0 .../+BpodLib/{+External => +external1}/+DataHash/uTest_DataHash.m | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename Functions/+BpodLib/{+External => +external1}/+DataHash/DataHash.m (100%) rename Functions/+BpodLib/{+External => +external1}/+DataHash/license.txt (100%) rename Functions/+BpodLib/{+External => +external1}/+DataHash/uTest_DataHash.m (100%) diff --git a/Functions/+BpodLib/+External/+DataHash/DataHash.m b/Functions/+BpodLib/+external1/+DataHash/DataHash.m similarity index 100% rename from Functions/+BpodLib/+External/+DataHash/DataHash.m rename to Functions/+BpodLib/+external1/+DataHash/DataHash.m diff --git a/Functions/+BpodLib/+External/+DataHash/license.txt b/Functions/+BpodLib/+external1/+DataHash/license.txt similarity index 100% rename from Functions/+BpodLib/+External/+DataHash/license.txt rename to Functions/+BpodLib/+external1/+DataHash/license.txt diff --git a/Functions/+BpodLib/+External/+DataHash/uTest_DataHash.m b/Functions/+BpodLib/+external1/+DataHash/uTest_DataHash.m similarity index 100% rename from Functions/+BpodLib/+External/+DataHash/uTest_DataHash.m rename to Functions/+BpodLib/+external1/+DataHash/uTest_DataHash.m From a2f3b17580b2ad813fcdfd1057056f666d6c66d9 Mon Sep 17 00:00:00 2001 From: George Stuyt Date: Wed, 29 May 2024 23:03:46 +1000 Subject: [PATCH 07/20] fix capitilisation pt 2 --- Functions/+BpodLib/{+external1 => +external}/+DataHash/DataHash.m | 0 .../+BpodLib/{+external1 => +external}/+DataHash/license.txt | 0 .../+BpodLib/{+external1 => +external}/+DataHash/uTest_DataHash.m | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename Functions/+BpodLib/{+external1 => +external}/+DataHash/DataHash.m (100%) rename Functions/+BpodLib/{+external1 => +external}/+DataHash/license.txt (100%) rename Functions/+BpodLib/{+external1 => +external}/+DataHash/uTest_DataHash.m (100%) diff --git a/Functions/+BpodLib/+external1/+DataHash/DataHash.m b/Functions/+BpodLib/+external/+DataHash/DataHash.m similarity index 100% rename from Functions/+BpodLib/+external1/+DataHash/DataHash.m rename to Functions/+BpodLib/+external/+DataHash/DataHash.m diff --git a/Functions/+BpodLib/+external1/+DataHash/license.txt b/Functions/+BpodLib/+external/+DataHash/license.txt similarity index 100% rename from Functions/+BpodLib/+external1/+DataHash/license.txt rename to Functions/+BpodLib/+external/+DataHash/license.txt diff --git a/Functions/+BpodLib/+external1/+DataHash/uTest_DataHash.m b/Functions/+BpodLib/+external/+DataHash/uTest_DataHash.m similarity index 100% rename from Functions/+BpodLib/+external1/+DataHash/uTest_DataHash.m rename to Functions/+BpodLib/+external/+DataHash/uTest_DataHash.m From 4a95fa82f251f8c4c6da0d08940b3a053387f786 Mon Sep 17 00:00:00 2001 From: George Stuyt Date: Wed, 29 May 2024 23:05:29 +1000 Subject: [PATCH 08/20] Remove Contents.m This will be added in the bigger +BpodLib PR --- Functions/+BpodLib/Contents.m | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 Functions/+BpodLib/Contents.m diff --git a/Functions/+BpodLib/Contents.m b/Functions/+BpodLib/Contents.m deleted file mode 100644 index 4ebcba73..00000000 --- a/Functions/+BpodLib/Contents.m +++ /dev/null @@ -1,4 +0,0 @@ -% The functiones, utilities, and helpers used by Bpod -% -% External - External functions used by Bpod -% Launcher - Protocol launch functions \ No newline at end of file From 0e4ad2d2af2b5d0a32068e57e632ba24c204483f Mon Sep 17 00:00:00 2001 From: George Stuyt Date: Wed, 29 May 2024 23:08:11 +1000 Subject: [PATCH 09/20] Remove forced addition of hash --- Functions/AddTrialEvents.m | 2 -- 1 file changed, 2 deletions(-) diff --git a/Functions/AddTrialEvents.m b/Functions/AddTrialEvents.m index 74190c9f..ecc096ab 100644 --- a/Functions/AddTrialEvents.m +++ b/Functions/AddTrialEvents.m @@ -65,8 +65,6 @@ TheTime = now; TE.Info.SessionStartTime_UTC = datestr(TheTime, 13); TE.Info.SessionStartTime_MATLAB = TheTime; - TE.Info.Protocol = BpodSystem.Path.CurrentProtocol; - TE.Info.MD5_Hash = BpodLib.Launcher.HashFile(BpodSystem.Path.CurrentProtocol); end TE.nTrials = TrialNum; %% Parse and add raw events for this trial From 59f17907a66cdcc6b0b2360e91cc2e158b13864a Mon Sep 17 00:00:00 2001 From: George Stuyt Date: Wed, 29 May 2024 23:12:37 +1000 Subject: [PATCH 10/20] Remove AddTrialEvents.m changes --- Functions/AddTrialEvents.m | 1 - 1 file changed, 1 deletion(-) diff --git a/Functions/AddTrialEvents.m b/Functions/AddTrialEvents.m index ecc096ab..fb4e80be 100644 --- a/Functions/AddTrialEvents.m +++ b/Functions/AddTrialEvents.m @@ -29,7 +29,6 @@ if isfield(TE, 'RawEvents') TrialNum = length(TE.RawEvents.Trial) + 1; else - % Intialise BpodSystem.Data.Info TrialNum = 1; TE.Info = struct; if BpodSystem.EmulatorMode == 1 From 240130c9eb65da91ec1515041677b5c99600f83d Mon Sep 17 00:00:00 2001 From: George Stuyt Date: Wed, 29 May 2024 23:31:01 +1000 Subject: [PATCH 11/20] Fix folder lowercase refactor --- Functions/+BpodLib/+external/+DataHash/uTest_DataHash.m | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Functions/+BpodLib/+external/+DataHash/uTest_DataHash.m b/Functions/+BpodLib/+external/+DataHash/uTest_DataHash.m index e6d817a8..cafd3395 100644 --- a/Functions/+BpodLib/+external/+DataHash/uTest_DataHash.m +++ b/Functions/+BpodLib/+external/+DataHash/uTest_DataHash.m @@ -187,10 +187,12 @@ function uTest_DataHash(doSpeed) % Run test on license file % Test added by GS in-lieu of 5th test in previous set -license_path = split(which('BpodLib.External.DataHash.DataHash'), ' '); +license_path = split(which('BpodLib.external.DataHash.DataHash'), ' '); +disp(license_path) license_path = license_path{1}; license_path = fullfile(fileparts(license_path), 'license.txt'); -assert(isfile(license_path), 'license.txt not found') + +assert(isfile(license_path), '%s not found', license_path) R = DataHash(license_path, 'file'); Want = 'c138996c2c7df49992ff044b25beb003'; From 82936c313873fd0e5fbc44436c5bfc91ba3c70b4 Mon Sep 17 00:00:00 2001 From: George Stuyt Date: Wed, 29 May 2024 23:31:09 +1000 Subject: [PATCH 12/20] Improve comments --- Functions/+BpodLib/+dataio/HashFile.m | 1 + 1 file changed, 1 insertion(+) diff --git a/Functions/+BpodLib/+dataio/HashFile.m b/Functions/+BpodLib/+dataio/HashFile.m index 23833749..822eeb63 100644 --- a/Functions/+BpodLib/+dataio/HashFile.m +++ b/Functions/+BpodLib/+dataio/HashFile.m @@ -2,6 +2,7 @@ %HashFile - Compute the hash of a file with MD5 % HASH = HashFile(filename) % The hash is returned as a string of hexadecimal digits computed using the MD5 message-digest algorithm. +% In windows the MD5 hash can be found with the command `certutil -hashfile filename MD5`. % % See also BpodLib.external.DataHash.DataHash. From 0ddf6c594d071d80032e902a2966a946329f9099 Mon Sep 17 00:00:00 2001 From: George Stuyt Date: Wed, 29 May 2024 23:55:03 +1000 Subject: [PATCH 13/20] Fix DataHash unit test Char is 16 bit, so hash test as 'File' and as 'ascii' (7 bit) won't work. Modified the test to only include the first 128 characters (2^7) --- .../+BpodLib/+external/+DataHash/uTest_DataHash.m | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Functions/+BpodLib/+external/+DataHash/uTest_DataHash.m b/Functions/+BpodLib/+external/+DataHash/uTest_DataHash.m index cafd3395..ac0beb67 100644 --- a/Functions/+BpodLib/+external/+DataHash/uTest_DataHash.m +++ b/Functions/+BpodLib/+external/+DataHash/uTest_DataHash.m @@ -6,7 +6,7 @@ function uTest_DataHash(doSpeed) % uTest_DataHash(doSpeed) % INPUT: % doSpeed: Optional logical flag to trigger time consuming speed tests. -% Default: TRUE. If no speed test is defined, this is ignored. +% Default: FALSE. If no speed test is defined, this is ignored. % OUTPUT: % On failure the test stops with an error. % The speed is compared to a Java method. @@ -19,6 +19,7 @@ function uTest_DataHash(doSpeed) % $File: Tools\UnitTests_\uTest_DataHash.m $ % History: % 001: 02-Mar-2019 19:22, First version. +% 002: 29-May-2024 23:33, Minor modifications from George Stuyt %#ok<*STRQUOT> % Accept string('s') for R2016b %#ok<*STRCLQT> @@ -148,11 +149,10 @@ function uTest_DataHash(doSpeed) '78901234567890'], ... '57edf4a22be3c955ac49da2e2107b67a'; ... ... -% char(0:255), 'e2c865db4162bed963bfaa9ef6ac18f0' -% certutils expects 'b1eba2174ca70416ae5819cb3659f929' -% DataHash gives 4df93029fb116c131c339e021f13698f reading a test.txt -% DataHash also give e1cb1402564d3f0d07fc946196789c81 on actual unittest -% Failure probably due to MATLAB char encoding differences + char(0:127), '37eff01866ba3f538421b30b7cbefcac'; ... +% char(0:255), 'e2c865db4162bed963bfaa9ef6ac18f0'; ... % Original from version 001 +% likely fails because matlab's char is 16bit, filehash and ascii hash are +% different }; try From e4c733b8f1bd1b6201b1840b670d057639469b2c Mon Sep 17 00:00:00 2001 From: George Stuyt Date: Wed, 29 May 2024 23:58:29 +1000 Subject: [PATCH 14/20] Add folder to fileHashes struct --- Functions/User Functions/SaveSessionVersion.m | 1 + 1 file changed, 1 insertion(+) diff --git a/Functions/User Functions/SaveSessionVersion.m b/Functions/User Functions/SaveSessionVersion.m index 226f58a3..b5e121c8 100644 --- a/Functions/User Functions/SaveSessionVersion.m +++ b/Functions/User Functions/SaveSessionVersion.m @@ -72,6 +72,7 @@ end fileindex = fileindex + 1; fileHashes(i).name = filepaths(i).name; + fileHashes(i).folder = filepaths(i).folder; fileHashes(i).hash = BpodLib.dataio.HashFile(fullfile(protocolpath, filepaths(i).name)); if p.Results.verbose fprintf('%s, ', filepaths(i).name); From b34df422ae61a914ed9ec9ece5054f767e18837c Mon Sep 17 00:00:00 2001 From: George Stuyt Date: Thu, 30 May 2024 00:03:58 +1000 Subject: [PATCH 15/20] Change location to Data.Info.VersionControl.FileHashes --- Functions/User Functions/SaveSessionVersion.m | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Functions/User Functions/SaveSessionVersion.m b/Functions/User Functions/SaveSessionVersion.m index b5e121c8..8f7bfd24 100644 --- a/Functions/User Functions/SaveSessionVersion.m +++ b/Functions/User Functions/SaveSessionVersion.m @@ -1,7 +1,7 @@ function hashOut = SaveSessionVersion(varargin) %SaveSessionVersion - Save current protocol files and their hashes into SessionData.Info.FileHashes % SaveSessionVersion(___) -% Using SaveSessionVersion within a protocol file will hash all files in the same directory as the protocol. +% Using SaveSessionVersion within a protocol file will hash all files in the same directory as the protocol into SessionData.Info.VersionControl.FileHashes % Hashes are unique to the file contents, so they can be used to determine if the protocol has changed. % Hashes are computed using the MD5 algorithm and recorded as hexadecimal strings. % @@ -84,7 +84,10 @@ end if p.Results.addtosessiondata - BpodSystem.Data.Info.FileHashes = fileHashes; + if ~isfield(BpodSystem.Data.Info, 'VersionControl') + BpodSystem.Data.Info.FileHashes = struct(); + end + BpodSystem.Data.Info.VersionControl.FileHashes = fileHashes; end if nargout > 0 From f724cb18d9cb3d381bf714959a6b1b75f3f5ae8e Mon Sep 17 00:00:00 2001 From: George Stuyt Date: Thu, 30 May 2024 00:07:44 +1000 Subject: [PATCH 16/20] Actually, Info.VersionControl.ProtocolFiles Other information could be added regarding the files --- Functions/User Functions/SaveSessionVersion.m | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Functions/User Functions/SaveSessionVersion.m b/Functions/User Functions/SaveSessionVersion.m index 8f7bfd24..06281e69 100644 --- a/Functions/User Functions/SaveSessionVersion.m +++ b/Functions/User Functions/SaveSessionVersion.m @@ -1,7 +1,7 @@ function hashOut = SaveSessionVersion(varargin) -%SaveSessionVersion - Save current protocol files and their hashes into SessionData.Info.FileHashes +%SaveSessionVersion - Save current protocol files and their hashes into SessionData.Info.VersionControl.ProtocolFiles % SaveSessionVersion(___) -% Using SaveSessionVersion within a protocol file will hash all files in the same directory as the protocol into SessionData.Info.VersionControl.FileHashes +% Using SaveSessionVersion within a protocol file will hash all files in the same directory as the protocol into SessionData.Info.VersionControl.ProtocolFiles. % Hashes are unique to the file contents, so they can be used to determine if the protocol has changed. % Hashes are computed using the MD5 algorithm and recorded as hexadecimal strings. % @@ -85,9 +85,9 @@ if p.Results.addtosessiondata if ~isfield(BpodSystem.Data.Info, 'VersionControl') - BpodSystem.Data.Info.FileHashes = struct(); + BpodSystem.Data.Info.VersionControl = struct(); end - BpodSystem.Data.Info.VersionControl.FileHashes = fileHashes; + BpodSystem.Data.Info.VersionControl.ProtocolFiles = fileHashes; end if nargout > 0 From 180861ad3ba7c021d7f41210a069c0e0db72dfd6 Mon Sep 17 00:00:00 2001 From: George Stuyt Date: Thu, 30 May 2024 00:25:49 +1000 Subject: [PATCH 17/20] Prevent overwriting .Info Other functions might insert information into .Info at protocol startup, but before this change it'd be overwritten --- Functions/AddTrialEvents.m | 4 +++- Functions/User Functions/SaveSessionVersion.m | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Functions/AddTrialEvents.m b/Functions/AddTrialEvents.m index fb4e80be..525b8169 100644 --- a/Functions/AddTrialEvents.m +++ b/Functions/AddTrialEvents.m @@ -30,7 +30,9 @@ TrialNum = length(TE.RawEvents.Trial) + 1; else TrialNum = 1; - TE.Info = struct; + if ~isfield(TE, 'Info') + TE.Info = struct; + end if BpodSystem.EmulatorMode == 1 TE.Info.StateMachineVersion = 'Bpod 0.7-1.0 EMULATOR'; else diff --git a/Functions/User Functions/SaveSessionVersion.m b/Functions/User Functions/SaveSessionVersion.m index 06281e69..9527fd84 100644 --- a/Functions/User Functions/SaveSessionVersion.m +++ b/Functions/User Functions/SaveSessionVersion.m @@ -84,6 +84,9 @@ end if p.Results.addtosessiondata + if ~isfield(BpodSystem.Data, 'Info') + BpodSystem.Data.Info = struct(); + end if ~isfield(BpodSystem.Data.Info, 'VersionControl') BpodSystem.Data.Info.VersionControl = struct(); end From b346f254b74ab012a330439a2faf30fd6f9971c6 Mon Sep 17 00:00:00 2001 From: George Stuyt Date: Thu, 30 May 2024 00:26:02 +1000 Subject: [PATCH 18/20] Add excludedExtensions --- Functions/User Functions/SaveSessionVersion.m | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Functions/User Functions/SaveSessionVersion.m b/Functions/User Functions/SaveSessionVersion.m index 9527fd84..0971c29a 100644 --- a/Functions/User Functions/SaveSessionVersion.m +++ b/Functions/User Functions/SaveSessionVersion.m @@ -10,6 +10,7 @@ % BpodSystem (struct): BpodSystem struct (default: global BpodSystem) % filepaths (cell): list of filepaths to hash (default: []) % protocolpath (char): path to the protocol (default: BpodSystem.Path.CurrentProtocol) +% excludedExtensions (cell): list of file extensions to exclude from hashing (default: {}) % verbose (logical): whether to print verbose output (default: false) % % Returns (optional): @@ -21,12 +22,15 @@ p.addParameter('BpodSystem', []); % allow overriding of BpodSystem for p.addParameter('filepaths', [], @iscell); p.addParameter('protocolpath', [], @ischar); +p.addParameter('excludedExtensions', {}, @iscell); p.addParameter('verbose', false, @islogical); p.parse(varargin{:}); % Prepare variables if isempty(p.Results.BpodSystem) global BpodSystem +else + BpodSystem = p.Results.BpodSystem; end if isempty(p.Results.protocolpath) @@ -57,8 +61,6 @@ end -excludedExtensions = {}; % todo: allow users to specify exclusions - % Hash files fileHashes = struct(); fileindex = 0; @@ -67,7 +69,7 @@ continue end [~, ~, ext] = fileparts(filepaths(i).name); - if ismember(ext, excludedExtensions) + if ismember(ext, p.Results.excludedExtensions) continue end fileindex = fileindex + 1; From 5a83011a77b2cde537cf32103fe04e09e84e0d7c Mon Sep 17 00:00:00 2001 From: George Stuyt Date: Thu, 30 May 2024 00:36:45 +1000 Subject: [PATCH 19/20] Add zip saving of protocol files --- Functions/User Functions/SaveSessionVersion.m | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/Functions/User Functions/SaveSessionVersion.m b/Functions/User Functions/SaveSessionVersion.m index 0971c29a..c54b9f86 100644 --- a/Functions/User Functions/SaveSessionVersion.m +++ b/Functions/User Functions/SaveSessionVersion.m @@ -6,11 +6,12 @@ % Hashes are computed using the MD5 algorithm and recorded as hexadecimal strings. % % Args: +% excludedExtensions (cell): list of file extensions to exclude from hashing (default: {}) % addtosessiondata (logical): whether to add the file hashes to the session data (default: true) -% BpodSystem (struct): BpodSystem struct (default: global BpodSystem) -% filepaths (cell): list of filepaths to hash (default: []) +% dozip (logical): whether to zip the protocol files (default: true) +% filepaths (cell): list of filepaths to hash (default: [], finds all files) % protocolpath (char): path to the protocol (default: BpodSystem.Path.CurrentProtocol) -% excludedExtensions (cell): list of file extensions to exclude from hashing (default: {}) +% BpodSystem (struct): BpodSystem struct (default: global BpodSystem) % verbose (logical): whether to print verbose output (default: false) % % Returns (optional): @@ -18,11 +19,12 @@ % Parse input p = inputParser(); +p.addParameter('excludedExtensions', {}, @iscell); p.addParameter('addtosessiondata', true, @islogical); -p.addParameter('BpodSystem', []); % allow overriding of BpodSystem for +p.addParameter('dozip', true, @islogical); p.addParameter('filepaths', [], @iscell); p.addParameter('protocolpath', [], @ischar); -p.addParameter('excludedExtensions', {}, @iscell); +p.addParameter('BpodSystem', []); % allow overriding of BpodSystem for testing purposes p.addParameter('verbose', false, @islogical); p.parse(varargin{:}); @@ -95,6 +97,15 @@ BpodSystem.Data.Info.VersionControl.ProtocolFiles = fileHashes; end +% Zip files +if p.Results.dozip + [savelocation, fname] = fileparts(BpodSystem.Path.CurrentDataFile); + zipfilename = fullfile(savelocation, [fname, '_protocol_files.zip']); + zip(zipfilename, fullfile(protocolpath, '*.*')); + if p.Results.verbose + fprintf('Zipped protocol files to %s\n', zipfilename); + end + if nargout > 0 hashOut = fileHashes; end From 0086c492b1c680751d2ab294715d3212dedfb5cb Mon Sep 17 00:00:00 2001 From: George Stuyt Date: Thu, 30 May 2024 00:44:08 +1000 Subject: [PATCH 20/20] Make files to zip match the list of files to hash --- Functions/User Functions/SaveSessionVersion.m | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/Functions/User Functions/SaveSessionVersion.m b/Functions/User Functions/SaveSessionVersion.m index c54b9f86..1ca9283b 100644 --- a/Functions/User Functions/SaveSessionVersion.m +++ b/Functions/User Functions/SaveSessionVersion.m @@ -54,15 +54,25 @@ filepaths = dir(fullfile(protocolpath, '*.*')); % look at top folders % remove dots filepaths = filepaths(~ismember({filepaths.name}, {'.', '..'})); + + % Print list of filenames concatenated + if p.Results.verbose + fprintf('Found %d files in %s:\n\t', length(filepaths), protocolpath); + end else - filepaths = p.Results.filepaths; -end -% Print list of filenames concatenated -if p.Results.verbose - fprintf('Found %d files in %s:\n\t', length(filepaths), protocolpath); + filepaths_cell = p.Results.filepaths; + % Turn into struct + filepaths = struct('folder', {}, 'name', {}); + for i = 1:length(filepaths_cell) + [folder, name, ext] = fileparts(filepaths_cell{i}); + filepaths(i).folder = folder; + filepaths(i).name = [name, ext]; + filepaths(i).isdir = isfolder(filepaths_cell{i}); + end end + % Hash files fileHashes = struct(); fileindex = 0; @@ -99,9 +109,11 @@ % Zip files if p.Results.dozip + + filepaths_cell = arrayfun(@(x) fullfile(x.folder, x.name), fileHashes, 'UniformOutput', false); [savelocation, fname] = fileparts(BpodSystem.Path.CurrentDataFile); zipfilename = fullfile(savelocation, [fname, '_protocol_files.zip']); - zip(zipfilename, fullfile(protocolpath, '*.*')); + zip(zipfilename, filepaths_cell); if p.Results.verbose fprintf('Zipped protocol files to %s\n', zipfilename); end