diff --git a/Functions/+BpodLib/+dataio/HashFile.m b/Functions/+BpodLib/+dataio/HashFile.m new file mode 100644 index 00000000..822eeb63 --- /dev/null +++ b/Functions/+BpodLib/+dataio/HashFile.m @@ -0,0 +1,15 @@ +function hash = HashFile(filename) +%HashFile - Compute the hash of a file with MD5 +% HASH = HashFile(filename) +% The hash is returned as a string of hexadecimal digits computed using the MD5 message-digest algorithm. +% In windows the MD5 hash can be found with the command `certutil -hashfile filename MD5`. +% +% See also BpodLib.external.DataHash.DataHash. + +% This function requires the DataHash function by Jan Simon. +% MATLAB FEX: http://www.mathworks.com/matlabcentral/fileexchange/31272 +% The DataHash function is distributed under the BSD license. + +hash = BpodLib.external.DataHash.DataHash(filename, 'file'); + +end \ No newline at end of file diff --git a/Functions/+BpodLib/+external/+DataHash/DataHash.m b/Functions/+BpodLib/+external/+DataHash/DataHash.m new file mode 100644 index 00000000..a234a73d --- /dev/null +++ b/Functions/+BpodLib/+external/+DataHash/DataHash.m @@ -0,0 +1,535 @@ +function Hash = DataHash(Data, varargin) +% DATAHASH - Checksum for Matlab array of any type +% This function creates a hash value for an input of any type. The type and +% dimensions of the input are considered as default, such that UINT8([0,0]) and +% UINT16(0) have different hash values. Nested STRUCTs and CELLs are parsed +% recursively. +% +% Hash = DataHash(Data, Opts...) +% INPUT: +% Data: Array of these built-in types: +% (U)INT8/16/32/64, SINGLE, DOUBLE, (real/complex, full/sparse) +% CHAR, LOGICAL, CELL (nested), STRUCT (scalar or array, nested), +% function_handle, string. +% Opts: Char strings to specify the method, the input and theoutput types: +% Input types: +% 'array': The contents, type and size of the input [Data] are +% considered for the creation of the hash. Nested CELLs +% and STRUCT arrays are parsed recursively. Empty arrays of +% different type reply different hashs. +% 'file': [Data] is treated as file name and the hash is calculated +% for the files contents. +% 'bin': [Data] is a numerical, LOGICAL or CHAR array. Only the +% binary contents of the array is considered, such that +% e.g. empty arrays of different type reply the same hash. +% 'ascii': Same as 'bin', but only the 8-bit ASCII part of the 16-bit +% Matlab CHARs is considered. +% Output types: +% 'hex', 'HEX': Lower/uppercase hexadecimal string. +% 'double', 'uint8': Numerical vector. +% 'base64': Base64. +% 'short': Base64 without padding. +% Hashing method: +% 'SHA-1', 'SHA-256', 'SHA-384', 'SHA-512', 'MD2', 'MD5'. +% Call DataHash without inputs to get a list of available methods. +% +% Default: 'MD5', 'hex', 'array' +% +% OUTPUT: +% Hash: String, DOUBLE or UINT8 vector. The length depends on the hashing +% method. +% If DataHash is called without inputs, a struct is replied: +% .HashVersion: Version number of the hashing method of this tool. In +% case of bugs or additions, the output can change. +% .Date: Date of release of the current HashVersion. +% .HashMethod: Cell string of the recognized hash methods. +% +% EXAMPLES: +% % Default: MD5, hex: +% DataHash([]) % 5b302b7b2099a97ba2a276640a192485 +% % MD5, Base64: +% DataHash(int32(1:10), 'short', 'MD5') % +tJN9yeF89h3jOFNN55XLg +% % SHA-1, Base64: +% S.a = uint8([]); +% S.b = {{1:10}, struct('q', uint64(415))}; +% DataHash(S, 'SHA-1', 'HEX') % 18672BE876463B25214CA9241B3C79CC926F3093 +% % SHA-1 of binary values: +% DataHash(1:8, 'SHA-1', 'bin') % 826cf9d3a5d74bbe415e97d4cecf03f445f69225 +% % SHA-256, consider ASCII part only (Matlab's CHAR has 16 bits!): +% DataHash('abc', 'SHA-256', 'ascii') +% % ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad +% % Or equivalently by converting the input to UINT8: +% DataHash(uint8('abc'), 'SHA-256', 'bin') +% +% NOTES: +% Function handles and user-defined objects cannot be converted uniquely: +% - The subfunction ConvertFuncHandle uses the built-in function FUNCTIONS, +% but the replied struct can depend on the Matlab version. +% - It is tried to convert objects to UINT8 streams in the subfunction +% ConvertObject. A conversion by STRUCT() might be more appropriate. +% Adjust these subfunctions on demand. +% +% MATLAB CHARs have 16 bits! Use Opt.Input='ascii' for comparisons with e.g. +% online hash generators. +% +% Matt Raum suggested this for e.g. user-defined objects: +% DataHash(getByteStreamFromArray(Data)) +% This works very well, but unfortunately getByteStreamFromArray is +% undocumented, such that it might vanish in the future or reply different +% output. +% +% For arrays the calculated hash value might be changed in new versions. +% Calling this function without inputs replies the version of the hash. +% +% The older style for input arguments is accepted also: Struct with fields +% 'Input', 'Method', 'OutFormat'. +% +% The C-Mex function GetMD5 is 2 to 100 times faster, but obtains MD5 only: +% http://www.mathworks.com/matlabcentral/fileexchange/25921 +% +% Tested: Matlab 2009a, 2015b(32/64), 2016b, 2018b, Win7/10 +% Author: Jan Simon, Heidelberg, (C) 2011-2019 matlab.2010(a)n(MINUS)simon.de +% +% See also: TYPECAST, CAST. +% +% Michael Kleder, "Compute Hash", no structs and cells: +% http://www.mathworks.com/matlabcentral/fileexchange/8944 +% Tim, "Serialize/Deserialize", converts structs and cells to a byte stream: +% http://www.mathworks.com/matlabcentral/fileexchange/29457 + +% $JRev: R-R V:043 Sum:VbfXFn6217Hp Date:18-Apr-2019 12:11:42 $ +% $License: BSD (use/copy/change/redistribute on own risk, mention the author) $ +% $UnitTest: uTest_DataHash $ +% $File: Tools\GLFile\DataHash.m $ +% History: +% 001: 01-May-2011 21:52, First version. +% 007: 10-Jun-2011 10:38, [Opt.Input], binary data, complex values considered. +% 011: 26-May-2012 15:57, Fixed: Failed for binary input and empty data. +% 014: 04-Nov-2012 11:37, Consider Mex-, MDL- and P-files also. +% Thanks to David (author 243360), who found this bug. +% Jan Achterhold (author 267816) suggested to consider Java objects. +% 016: 01-Feb-2015 20:53, Java heap space exhausted for large files. +% Now files are process in chunks to save memory. +% 017: 15-Feb-2015 19:40, Collsions: Same hash for different data. +% Examples: zeros(1,1) and zeros(1,1,0) +% complex(0) and zeros(1,1,0,0) +% Now the number of dimensions is included, to avoid this. +% 022: 30-Mar-2015 00:04, Bugfix: Failed for strings and [] without TYPECASTX. +% Ross found these 2 bugs, which occur when TYPECASTX is not installed. +% If you need the base64 format padded with '=' characters, adjust +% fBase64_enc as you like. +% 026: 29-Jun-2015 00:13, Changed hash for STRUCTs. +% Struct arrays are analysed field by field now, which is much faster. +% 027: 13-Sep-2015 19:03, 'ascii' input as abbrev. for Input='bin' and UINT8(). +% 028: 15-Oct-2015 23:11, Example values in help section updated to v022. +% 029: 16-Oct-2015 22:32, Use default options for empty input. +% 031: 28-Feb-2016 15:10, New hash value to get same reply as GetMD5. +% New Matlab version (at least 2015b) use a fast method for TYPECAST, such +% that calling James Tursa's TYPECASTX is not needed anymore. +% Matlab 6.5 not supported anymore: MException for CATCH. +% 033: 18-Jun-2016 14:28, BUGFIX: Failed on empty files. +% Thanks to Christian (AuthorID 2918599). +% 035: 19-May-2018 01:11, STRING type considered. +% 040: 13-Nov-2018 01:20, Fields of Opt not case-sensitive anymore. +% 041: 09-Feb-2019 18:12, ismethod(class(V),) to support R2018b. +% 042: 02-Mar-2019 18:39, base64: in Java, short: Base64 with padding. +% Unit test. base64->short. + +% OPEN BUGS: +% Nath wrote: +% function handle refering to struct containing the function will create +% infinite loop. Is there any workaround ? +% Example: +% d= dynamicprops(); +% addprop(d,'f'); +% d.f= @(varargin) struct2cell(d); +% DataHash(d.f) % infinite loop +% This is caught with an error message concerning the recursion limit now. + +%#ok<*CHARTEN> + +% Reply current version if called without inputs: ------------------------------ +if nargin == 0 + R = Version_L; + + if nargout == 0 + disp(R); + else + Hash = R; + end + + return; +end + +% Parse inputs: ---------------------------------------------------------------- +[Method, OutFormat, isFile, isBin, Data] = ParseInput(Data, varargin{:}); + +% Create the engine: ----------------------------------------------------------- +try + Engine = java.security.MessageDigest.getInstance(Method); + +catch ME % Handle errors during initializing the engine: + if ~usejava('jvm') + Error_L('needJava', 'DataHash needs Java.'); + end + Error_L('BadInput2', 'Invalid hashing algorithm: [%s]. %s', ... + Method, ME.message); +end + +% Create the hash value: ------------------------------------------------------- +if isFile + [FID, Msg] = fopen(Data, 'r'); % Open the file + if FID < 0 + Error_L('BadFile', ['Cannot open file: %s', char(10), '%s'], Data, Msg); + end + + % Read file in chunks to save memory and Java heap space: + Chunk = 1e6; % Fastest for 1e6 on Win7/64, HDD + Count = Chunk; % Dummy value to satisfy WHILE condition + while Count == Chunk + [Data, Count] = fread(FID, Chunk, '*uint8'); + if Count ~= 0 % Avoid error for empty file + Engine.update(Data); + end + end + fclose(FID); + +elseif isBin % Contents of an elementary array, type tested already: + if ~isempty(Data) % Engine.update fails for empty input! + if isnumeric(Data) + if isreal(Data) + Engine.update(typecast(Data(:), 'uint8')); + else + Engine.update(typecast(real(Data(:)), 'uint8')); + Engine.update(typecast(imag(Data(:)), 'uint8')); + end + elseif islogical(Data) % TYPECAST cannot handle LOGICAL + Engine.update(typecast(uint8(Data(:)), 'uint8')); + elseif ischar(Data) % TYPECAST cannot handle CHAR + Engine.update(typecast(uint16(Data(:)), 'uint8')); + % Bugfix: Line removed + elseif myIsString(Data) + if isscalar(Data) + Engine.update(typecast(uint16(Data{1}), 'uint8')); + else + Error_L('BadBinData', 'Bin type requires scalar string.'); + end + else % This should have been caught above! + Error_L('BadBinData', 'Data type not handled: %s', class(Data)); + end + end +else % Array with type: + Engine = CoreHash(Data, Engine); +end + +% Calculate the hash: ---------------------------------------------------------- +Hash = typecast(Engine.digest, 'uint8'); + +% Convert hash specific output format: ----------------------------------------- +switch OutFormat + case 'hex' + Hash = sprintf('%.2x', double(Hash)); + case 'HEX' + Hash = sprintf('%.2X', double(Hash)); + case 'double' + Hash = double(reshape(Hash, 1, [])); + case 'uint8' + Hash = reshape(Hash, 1, []); + case 'short' + Hash = fBase64_enc(double(Hash), 0); + case 'base64' + Hash = fBase64_enc(double(Hash), 1); + + otherwise + Error_L('BadOutFormat', ... + '[Opt.Format] must be: HEX, hex, uint8, double, base64.'); +end + +end + +% ****************************************************************************** +function Engine = CoreHash(Data, Engine) + +% Consider the type and dimensions of the array to distinguish arrays with the +% same data, but different shape: [0 x 0] and [0 x 1], [1,2] and [1;2], +% DOUBLE(0) and SINGLE([0,0]): +% < v016: [class, size, data]. BUG! 0 and zeros(1,1,0) had the same hash! +% >= v016: [class, ndims, size, data] +Engine.update([uint8(class(Data)), ... + typecast(uint64([ndims(Data), size(Data)]), 'uint8')]); + +if issparse(Data) % Sparse arrays to struct: + [S.Index1, S.Index2, S.Value] = find(Data); + Engine = CoreHash(S, Engine); +elseif isstruct(Data) % Hash for all array elements and fields: + F = sort(fieldnames(Data)); % Ignore order of fields + for iField = 1:length(F) % Loop over fields + aField = F{iField}; + Engine.update(uint8(aField)); + for iS = 1:numel(Data) % Loop over elements of struct array + Engine = CoreHash(Data(iS).(aField), Engine); + end + end +elseif iscell(Data) % Get hash for all cell elements: + for iS = 1:numel(Data) + Engine = CoreHash(Data{iS}, Engine); + end +elseif isempty(Data) % Nothing to do +elseif isnumeric(Data) + if isreal(Data) + Engine.update(typecast(Data(:), 'uint8')); + else + Engine.update(typecast(real(Data(:)), 'uint8')); + Engine.update(typecast(imag(Data(:)), 'uint8')); + end +elseif islogical(Data) % TYPECAST cannot handle LOGICAL + Engine.update(typecast(uint8(Data(:)), 'uint8')); +elseif ischar(Data) % TYPECAST cannot handle CHAR + Engine.update(typecast(uint16(Data(:)), 'uint8')); +elseif myIsString(Data) % [19-May-2018] String class in >= R2016b + classUint8 = uint8([117, 105, 110, 116, 49, 54]); % 'uint16' + for iS = 1:numel(Data) + % Emulate without recursion: Engine = CoreHash(uint16(Data{iS}), Engine) + aString = uint16(Data{iS}); + Engine.update([classUint8, ... + typecast(uint64([ndims(aString), size(aString)]), 'uint8')]); + if ~isempty(aString) + Engine.update(typecast(uint16(aString), 'uint8')); + end + end + +elseif isa(Data, 'function_handle') + Engine = CoreHash(ConvertFuncHandle(Data), Engine); +elseif (isobject(Data) || isjava(Data)) && ismethod(class(Data), 'hashCode') + Engine = CoreHash(char(Data.hashCode), Engine); +else % Most likely a user-defined object: + try + BasicData = ConvertObject(Data); + catch ME + error(['JSimon:', mfilename, ':BadDataType'], ... + '%s: Cannot create elementary array for type: %s\n %s', ... + mfilename, class(Data), ME.message); + end + + try + Engine = CoreHash(BasicData, Engine); + catch ME + if strcmpi(ME.identifier, 'MATLAB:recursionLimit') + ME = MException(['JSimon:', mfilename, ':RecursiveType'], ... + '%s: Cannot create hash for recursive data type: %s', ... + mfilename, class(Data)); + end + throw(ME); + end +end + +end + +% ****************************************************************************** +function [Method, OutFormat, isFile, isBin, Data] = ParseInput(Data, varargin) + +% Default options: ------------------------------------------------------------- +Method = 'MD5'; +OutFormat = 'hex'; +isFile = false; +isBin = false; + +% Check number and type of inputs: --------------------------------------------- +nOpt = nargin - 1; +Opt = varargin; +if nOpt == 1 && isa(Opt{1}, 'struct') % Old style Options as struct: + Opt = struct2cell(Opt{1}); + nOpt = numel(Opt); +end + +% Loop over strings in the input: ---------------------------------------------- +for iOpt = 1:nOpt + aOpt = Opt{iOpt}; + if ~ischar(aOpt) + Error_L('BadInputType', '[Opt] must be a struct or chars.'); + end + + switch lower(aOpt) + case 'file' % Data contains the file name: + isFile = true; + case {'bin', 'binary'} % Just the contents of the data: + if (isnumeric(Data) || ischar(Data) || islogical(Data) || ... + myIsString(Data)) == 0 || issparse(Data) + Error_L('BadDataType', ['[Bin] input needs data type: ', ... + 'numeric, CHAR, LOGICAL, STRING.']); + end + isBin = true; + case 'array' + isBin = false; % Is the default already + case {'asc', 'ascii'} % 8-bit part of MATLAB CHAR or STRING: + isBin = true; + if ischar(Data) + Data = uint8(Data); + elseif myIsString(Data) && numel(Data) == 1 + Data = uint8(char(Data)); + else + Error_L('BadDataType', ... + 'ASCII method: Data must be a CHAR or scalar STRING.'); + end + case 'hex' + if aOpt(1) == 'H' + OutFormat = 'HEX'; + else + OutFormat = 'hex'; + end + case {'double', 'uint8', 'short', 'base64'} + OutFormat = lower(aOpt); + otherwise % Guess that this is the method: + Method = upper(aOpt); + end +end + +end + +% ****************************************************************************** +function FuncKey = ConvertFuncHandle(FuncH) +% The subfunction ConvertFuncHandle converts function_handles to a struct +% using the Matlab function FUNCTIONS. The output of this function changes +% with the Matlab version, such that DataHash(@sin) replies different hashes +% under Matlab 6.5 and 2009a. +% An alternative is using the function name and name of the file for +% function_handles, but this is not unique for nested or anonymous functions. +% If the MATLABROOT is removed from the file's path, at least the hash of +% Matlab's toolbox functions is (usually!) not influenced by the version. +% Finally I'm in doubt if there is a unique method to hash function handles. +% Please adjust the subfunction ConvertFuncHandles to your needs. + +% The Matlab version influences the conversion by FUNCTIONS: +% 1. The format of the struct replied FUNCTIONS is not fixed, +% 2. The full paths of toolbox function e.g. for @mean differ. +FuncKey = functions(FuncH); + +% Include modification file time and file size. Suggested by Aslak Grinsted: +if ~isempty(FuncKey.file) + d = dir(FuncKey.file); + if ~isempty(d) + FuncKey.filebytes = d.bytes; + FuncKey.filedate = d.datenum; + end +end + +% ALTERNATIVE: Use name and path. The part of the toolbox functions +% is replaced such that the hash for @mean does not depend on the Matlab +% version. +% Drawbacks: Anonymous functions, nested functions... +% funcStruct = functions(FuncH); +% funcfile = strrep(funcStruct.file, matlabroot, ''); +% FuncKey = uint8([funcStruct.function, ' ', funcfile]); + +% Finally I'm afraid there is no unique method to get a hash for a function +% handle. Please adjust this conversion to your needs. + +end + +% ****************************************************************************** +function DataBin = ConvertObject(DataObj) +% Convert a user-defined object to a binary stream. There cannot be a unique +% solution, so this part is left for the user... + +try % Perhaps a direct conversion is implemented: + DataBin = uint8(DataObj); + + % Matt Raum had this excellent idea - unfortunately this function is + % undocumented and might not be supported in te future: + % DataBin = getByteStreamFromArray(DataObj); + +catch % Or perhaps this is better: + WarnS = warning('off', 'MATLAB:structOnObject'); + DataBin = struct(DataObj); + warning(WarnS); +end + +end + +% ****************************************************************************** +function Out = fBase64_enc(In, doPad) +% Encode numeric vector of UINT8 values to base64 string. + +B64 = org.apache.commons.codec.binary.Base64; +% https://au.mathworks.com/matlabcentral/fileexchange/31272-datahash?tab=discussions#discussions_2595901 +Out = char(B64.encode(In)).'; +if ~doPad + Out(Out == '=') = []; +end + +% Matlab method: +% Pool = [65:90, 97:122, 48:57, 43, 47]; % [0:9, a:z, A:Z, +, /] +% v8 = [128; 64; 32; 16; 8; 4; 2; 1]; +% v6 = [32, 16, 8, 4, 2, 1]; +% +% In = reshape(In, 1, []); +% X = rem(floor(bsxfun(@rdivide, In, v8)), 2); +% d6 = rem(numel(X), 6); +% if d6 ~= 0 +% X = [X(:); zeros(6 - d6, 1)]; +% end +% Out = char(Pool(1 + v6 * reshape(X, 6, []))); +% +% p = 3 - rem(numel(Out) - 1, 4); +% if doPad && p ~= 0 % Standard base64 string with trailing padding: +% Out = [Out, repmat('=', 1, p)]; +% end + +end + +% ****************************************************************************** +function T = myIsString(S) +% isstring was introduced in R2016: +persistent hasString +if isempty(hasString) + matlabVer = [100, 1] * sscanf(version, '%d.', 2); + hasString = (matlabVer >= 901); % isstring existing since R2016b +end + +T = hasString && isstring(S); % Short-circuting + +end + +% ****************************************************************************** +function R = Version_L() +% The output differs between versions of this function. So give the user a +% chance to recognize the version: +% 1: 01-May-2011, Initial version +% 2: 15-Feb-2015, The number of dimensions is considered in addition. +% In version 1 these variables had the same hash: +% zeros(1,1) and zeros(1,1,0), complex(0) and zeros(1,1,0,0) +% 3: 29-Jun-2015, Struct arrays are processed field by field and not element +% by element, because this is much faster. In consequence the hash value +% differs, if the input contains a struct. +% 4: 28-Feb-2016 15:20, same output as GetMD5 for MD5 sums. Therefore the +% dimensions are casted to UINT64 at first. +% 19-May-2018 01:13, STRING type considered. +R.HashVersion = 4; +R.Date = [2018, 5, 19]; + +R.HashMethod = {}; +try + Provider = java.security.Security.getProviders; + for iProvider = 1:numel(Provider) + S = char(Provider(iProvider).getServices); + Index = strfind(S, 'MessageDigest.'); + for iDigest = 1:length(Index) + Digest = strtok(S(Index(iDigest):end)); + Digest = strrep(Digest, 'MessageDigest.', ''); + R.HashMethod = cat(2, R.HashMethod, {Digest}); + end + end +catch ME + fprintf(2, '%s\n', ME.message); + R.HashMethod = 'error'; +end + +end + +% ****************************************************************************** +function Error_L(ID, varargin) + +error(['JSimon:', mfilename, ':', ID], ['*** %s: ', varargin{1}], ... + mfilename, varargin{2:nargin - 1}); + +end diff --git a/Functions/+BpodLib/+external/+DataHash/license.txt b/Functions/+BpodLib/+external/+DataHash/license.txt new file mode 100644 index 00000000..981079dd --- /dev/null +++ b/Functions/+BpodLib/+external/+DataHash/license.txt @@ -0,0 +1,25 @@ +Copyright (c) 2018-2019, Jan Simon +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution +* Neither the name of nor the names of its + contributors may be used to endorse or promote products derived from this + software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Functions/+BpodLib/+external/+DataHash/uTest_DataHash.m b/Functions/+BpodLib/+external/+DataHash/uTest_DataHash.m new file mode 100644 index 00000000..ac0beb67 --- /dev/null +++ b/Functions/+BpodLib/+external/+DataHash/uTest_DataHash.m @@ -0,0 +1,394 @@ +function uTest_DataHash(doSpeed) +% Automatic test: DataHash (Mex) +% This is a routine for automatic testing. It is not needed for processing and +% can be deleted or moved to a folder, where it does not bother. +% +% uTest_DataHash(doSpeed) +% INPUT: +% doSpeed: Optional logical flag to trigger time consuming speed tests. +% Default: FALSE. If no speed test is defined, this is ignored. +% OUTPUT: +% On failure the test stops with an error. +% The speed is compared to a Java method. +% +% Tested: Matlab 2009a, 2015b(32/64), 2016b, 2018b, Win7/10 +% Author: Jan Simon, Heidelberg, (C) 2009-2019 matlab.2010(a)n(MINUS)simon.de + +% $JRev: R-e V:004 Sum:cYYIAiiAf7sM Date:19-May-2019 16:58:59 $ +% $License: BSD (use/copy/change/redistribute on own risk, mention the author) $ +% $File: Tools\UnitTests_\uTest_DataHash.m $ +% History: +% 001: 02-Mar-2019 19:22, First version. +% 002: 29-May-2024 23:33, Minor modifications from George Stuyt + +%#ok<*STRQUOT> % Accept string('s') for R2016b +%#ok<*STRCLQT> + +import BpodLib.external.DataHash.DataHash + +% Initialize: ================================================================== +% Global Interface: ------------------------------------------------------------ +ErrID = ['JSimon:', mfilename]; + +MatlabV = [100, 1] * sscanf(version, '%d.', 2); + +% Initial values: -------------------------------------------------------------- +if nargin == 0 + doSpeed = false; +end + +% Program Interface: ----------------------------------------------------------- +% User Interface: -------------------------------------------------------------- +% Do the work: ================================================================= +fprintf('==== Test DataHash: %s\n', datestr(now, 0)); +fprintf(' Matlab: %s\n', version); + +fprintf(' Java: %s\n\n', version('-java')); + +% Known answer tests - see RFC1321: -------------------------------------------- +disp('== Known answer tests:'); + +S.a = uint8([]); +S.b = {{1:10}, struct('q', uint64(415))}; + +TestData = { ... + ... % Desc, Data, Opt, Result: + '[]', [], {}, '5b302b7b2099a97ba2a276640a192485'; ... + ... + 'int32(1:10), short, MD5', int32(1:10), {'short', 'MD5'}, ... + '+tJN9yeF89h3jOFNN55XLg'; ... + ... + 'int32(1:10), short, MD5, Opt as struct', int32(1:10), ... + {struct('Format', 'short', 'Method', 'MD5')}, ... + '+tJN9yeF89h3jOFNN55XLg'; ... + ... + 'Struct, HEX, SHA-1', S, ... + {'HEX', 'SHA-1'}, '18672BE876463B25214CA9241B3C79CC926F3093'; ... + ... + 'Binary, SHA-1', 1:8, {'SHA-1', 'bin'}, ... + '826cf9d3a5d74bbe415e97d4cecf03f445f69225'; ... + ... + '''abc'', SHA-256, ASCII', 'abc', {'SHA-256', 'ascii'}, ... + 'ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad'; ... + ... + 'uint8(''abc''), SHA-256, ASCII', uint8('abc'), {'SHA-256', 'bin'}, ... + 'ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad'; ... + ... + 'message digest, MD5, bin', 'message digest', {'MD5', 'ascii'}, ... + 'f96b697d7cb7938d525a2f31aaf161d0'; ... + ... + 'char(0:255), MD5, bin', char(0:255), {'MD5', 'ascii'}, ... + 'e2c865db4162bed963bfaa9ef6ac18f0'; ... + ... + 'char(0:255), MD5, bin', char(0:255), {'MD5', 'Array'}, ... + '1e4558b49c05611cdb280a79cb2dbe34'; ... + ... + '[], SHA-256, base64', 1:33, {'SHA-256', 'base64'}, ... + 'SuuPZKpce2KetxeIClt1EXz/mCztEuYiawG9vaxKcfc='; ... + ... + '[], SHA-256, base64', 1:33, {'SHA-512', 'base64'}, ... + ['V9Yp/ZBUbfmzQZ7WRzRvqNYLbb6Kzgrc3iaqPH7ta8MS/bMPKc7j', ... + '+FRV5Oexu3OCOQ1+2p/E+ZcgKyRHheq0kQ==']; ... + ... + '[], MD5, short', 1:33, {'MD5', 'short'}, ... + 'RoNguVVzgq6s7ll9xoCSSg'; ... + ... + '[], SHA-256, short', 1:33, {'SHA-512', 'short'}, ... + ['V9Yp/ZBUbfmzQZ7WRzRvqNYLbb6Kzgrc3iaqPH7ta8MS/bMPKc7j', ... + '+FRV5Oexu3OCOQ1+2p/E+ZcgKyRHheq0kQ']; ... + ... + }; + +% Create string arrays, if possible: +if MatlabV >= 901 % R2016b + TestData = cat(1, TestData, { ... + '"", array', string(''), {'Array'}, ... + '061bdd545213c6a236e0f3d655e38ff4'; ... + ... + '"hello", array', string('hello'), {'Array'}, ... + '2614526bcbd4af5a8e7bf79d1d0d92ab'; ... + ... + '["hello", "world"]', string({'hello', 'world'}), {'Array'}, ... + 'a1bdbbe9a15c249764847ead9bf47326'; ... + ... + '["hello"; ""; "world"]', string({'hello'; ''; 'world'}), {'Array'}, ... + 'a6df2dc811d4e8dab214c01ce0dfc4b9'; ... + ... + '"", ascii', string(''), {'ascii'}, ... + 'd41d8cd98f00b204e9800998ecf8427e'; ... + ... + '"hello", ascii', string('hello'), {'ascii'}, ... + '5d41402abc4b2a76b9719d911017c592'; ... + ... + }); +end + +% Run the known answer tests: +for iTest = 1:size(TestData, 1) + Test = TestData(iTest, :); + R = DataHash(Test{2}, Test{3}{:}); + if isequal(R, Test{4}) + fprintf(' ok: %s\n', Test{1}); + else + fprintf(2, 'Want: %s\nGot: %s\n', Test{4}, R); + error([ErrID, ':KAT'], 'Failed: %s', Test{1}); + end +end + +% Create test file: +TestFile = fullfile(tempdir, [mfilename, '.txt']); +TestData = {'', 'd41d8cd98f00b204e9800998ecf8427e'; ... + ... + 'abcdefghijklmnopqrstuvwxyz', ... + 'c3fcd3d76192e4007dfb496cca67e13b'; ... + ... + 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789', ... + 'd174ab98d277d9f5a5611c2c9f419d9f'; ... + ... + ['123456789012345678901234567890123456789012345678901234567890123456', ... + '78901234567890'], ... + '57edf4a22be3c955ac49da2e2107b67a'; ... + ... + char(0:127), '37eff01866ba3f538421b30b7cbefcac'; ... +% char(0:255), 'e2c865db4162bed963bfaa9ef6ac18f0'; ... % Original from version 001 +% likely fails because matlab's char is 16bit, filehash and ascii hash are +% different + }; + +try + for iTest = 1:size(TestData, 1) + Test = TestData(iTest, :); + + % Create the file: + [fid, msg] = fopen(TestFile, 'w'); + assert(fid ~= -1, msg); + fwrite(fid, Test{1}); + fclose(fid); + + % Get hash for the file: + R = DataHash(TestFile, 'File'); + Want = DataHash(Test{1}, 'ascii'); + if isequal(R, Want, Test{2}) + fprintf(' ok: empty file\n'); + else + fprintf(2, 'Known: %s\nHashed: %s\nGot: %s\n', Test{2}, Want, R); + error([ErrID, ':KAT'], 'Failed: File access'); + end + end + +catch ME + if exist(TestFile, 'file') + delete(TestFile); + end + rethrow(ME); +end + +delete(TestFile); + +% Run test on license file +% Test added by GS in-lieu of 5th test in previous set +license_path = split(which('BpodLib.external.DataHash.DataHash'), ' '); +disp(license_path) +license_path = license_path{1}; +license_path = fullfile(fileparts(license_path), 'license.txt'); + +assert(isfile(license_path), '%s not found', license_path) + +R = DataHash(license_path, 'file'); +Want = 'c138996c2c7df49992ff044b25beb003'; +if isequal(R, Want) + fprintf(' ok: +DataHash/license.txt\n') +else + fprintf(2, 'Want: %s\nGot: %s\n', Want, R); + error([ErrID, ':KAT'], 'Failed: File access'); +end + + + +% Check different output types: ------------------------------------------------ +N = 1000; +B64 = org.apache.commons.codec.binary.Base64; + +disp('== Check output types:'); +for i = 1:N + data = uint8(fix(rand(1, 1 + fix(rand * 100)) * 256)); + lowHexOut = DataHash(data, 'bin', 'hex'); + upHexOut = DataHash(data, 'bin', 'HEX'); + decOut = DataHash(data, 'bin', 'Double'); + base64Out = DataHash(data, 'bin', 'Base64'); + shortOut = DataHash(data, 'bin', 'short'); + uint8Out = DataHash(data, 'bin', 'Uint8'); + + base64pad = char(B64.encode(decOut)).'; + base64short = strrep(base64pad, '=', ''); + + if not(strcmpi(lowHexOut, upHexOut) && ... + isSame(sscanf(lowHexOut, '%2x'), decOut(:)) && ... + isSame(base64Out, base64pad) && ... + isSame(shortOut, base64short) && ... + isSame(uint8Out, uint8(decOut))) + fprintf('\n'); + error([ErrID, ':Output'], 'Different results for output types.'); + end + + % Check binary, e.g. if the data length is a multiple of 2: + if rem(length(data), 2) == 0 + doubleData = double(data); + uniData = char(doubleData(1:2:end) + 256 * doubleData(2:2:end)); + uniOut = DataHash(uniData, 'binary', 'double'); + if not(isequal(uniOut, decOut)) + error([ErrID, ':Output'], 'Different results for binary mode.'); + end + end +end +fprintf([' ok: %d random tests with hex, HEX, double, uint8, base64 ', ... + 'output\n'], N); + +% Check arrays as inputs: ------------------------------------------------------ +disp('== Test array input:'); + +% Hash must depend on the type of the array: +S1 = DataHash([], 'Array'); +if ~isequal(S1, '5b302b7b2099a97ba2a276640a192485') + error([ErrID, ':Array'], 'Bad result for array: []'); +end + +S1 = DataHash(uint8([]), 'Array'); +if ~isequal(S1, 'cb8a2273d1168a72b70833bb0d79be13') + error([ErrID, ':Array'], 'Bad result for array: uint8([])'); +end + +S1 = DataHash(int8([]), 'Array'); +if ~isequal(S1, '0160dd4473fe1a952572be239e077ed3') + error([ErrID, ':Array'], 'Bad result for array: int8([])'); +end + +Data = struct('Field1', 'string', 'Field2', {{'Cell string', '2nd string'}}); +Data.Field3 = Data; +S1 = DataHash(Data, 'Array'); +if ~isequal(S1, '4fe320b06e3aaaf4ba712980d649e274') + error([ErrID, ':Array'], 'Bad result for array: .'); +end + +Data = sparse([1,0,2; 0,3,0; 4, 0,0]); +S1 = DataHash(Data, 'Array'); +if ~isequal(S1, 'f157bdc9173dff169c782dd639984c82') + error([ErrID, ':Array'], 'Bad result for array: .'); +end +fprintf(' ok: Array\n'); + +% Uninitialized cells contain NULL pointers: +Data = cell(1, 2); +S1 = DataHash(Data, 'Array'); +if ~isequal(S1, '161842037bc65b9f3bceffdeb4a8d3bd') + error([ErrID, ':Array'], 'Bad result for {NULL, NULL}.'); +end +fprintf(' ok: Null pointer\n'); + +% Check string type: +if MatlabV >= 901 % R2016b + Data = string('hello'); + S1 = DataHash(Data, 'Array'); + if ~isequal(S1, '2614526bcbd4af5a8e7bf79d1d0d92ab') + error([ErrID, ':String'], 'Bad result for string.'); + end + + Data = string({'hello', 'world'}); + S1 = DataHash(Data, 'Array'); + if ~isequal(S1, 'a1bdbbe9a15c249764847ead9bf47326') + error([ErrID, ':String'], 'Bad result for string.'); + end + fprintf(' ok: String class\n'); +end + +% Speed test: ------------------------------------------------------------------ +if doSpeed + disp('== Test speed:'); + disp(' * Slower for shorter data due to overhead of calling a function!'); + disp(' * Process data in memory, not from disk'); + Delay = 2; + + % Compare speed with the C-mex GetMD5, if available: + getmd5_M = which('GetMD5.m'); + getmd5_X = which(['GetMD5.', mexext]); + if ~isempty(getmd5_M) && ~isempty(getmd5_X) + Str = fileread(getmd5_M); + hasGetMD5 = any(strfind(Str, 'Author: Jan Simon')); + else + hasGetMD5 = false; % [BUGFIX] 17-May-2019, Thanks zmi zmi + end + + if hasGetMD5 + fprintf(' * Compare with: %s\n', getmd5_X); + fprintf(' * DataHash uses Java for the hashing, GetMD5 fast C code\n\n'); + fprintf(' Data size: DataHash: GetMD5:\n'); + else + fprintf('\n Data size: DataHash:\n'); + end + + for Len = [10, 100, 1000, 10000, 1e5, 1e6, 1e7, 1e8] + [Number, Unit] = UnitPrint(Len, false); + fprintf('%12s ', [Number, ' ', Unit]); + data = uint8(fix(rand(1, Len) * 256)); + + % Measure time: + iLoop = 0; + Time = 0; + tic; + while Time < Delay || iLoop < 2 + Hash = DataHash(data, 'binary', 'uint8'); + iLoop = iLoop + 1; + Time = toc; + end + LoopPerSec = iLoop / Time; + [Number, Unit] = UnitPrint(LoopPerSec * Len, true); + + if hasGetMD5 % Compare with GetMD5, if available: + iLoop = 0; + Time = 0; + tic; + while Time < Delay || iLoop < 2 + Hash2 = GetMD5(data, 'binary', 'uint8'); + iLoop = iLoop + 1; + Time = toc; + end + LoopPerSec2 = iLoop / Time; + [Number2, Unit2] = UnitPrint(LoopPerSec2 * Len, true); + + fprintf('%8s %s/s %9s %s/s\n', Number, Unit, Number2, Unit2); + + % Compare the results: + if ~isequal(Hash, Hash2) + error([ErrID, ':Compare'], 'Result differs from GetMD5.'); + end + + else + fprintf('%8s %s/s\n', Number, Unit); + end + end +end + +fprintf('\n== DataHash passed the tests.\n'); + +end + +% ****************************************************************************** +function E = isSame(A, B) +E = isequal(A, B) && strcmp(class(A), class(B)); +end + +% ****************************************************************************** +function [Number, Unit] = UnitPrint(N, useMB) + +if N >= 1e6 || useMB + Number = sprintf('%.1f', N / 1e6); + Unit = 'MB'; +elseif N >= 1e3 + Number = sprintf('%.1f', N / 1000); + Unit = 'kB'; +else + Number = sprintf('%g', N); + Unit = ' B'; +end + +end diff --git a/Functions/AddTrialEvents.m b/Functions/AddTrialEvents.m index fb4e80be..525b8169 100644 --- a/Functions/AddTrialEvents.m +++ b/Functions/AddTrialEvents.m @@ -30,7 +30,9 @@ TrialNum = length(TE.RawEvents.Trial) + 1; else TrialNum = 1; - TE.Info = struct; + if ~isfield(TE, 'Info') + TE.Info = struct; + end if BpodSystem.EmulatorMode == 1 TE.Info.StateMachineVersion = 'Bpod 0.7-1.0 EMULATOR'; else diff --git a/Functions/Launch manager/NewLaunchManager.m b/Functions/Launch manager/NewLaunchManager.m index 124e9a99..d3d189b0 100644 --- a/Functions/Launch manager/NewLaunchManager.m +++ b/Functions/Launch manager/NewLaunchManager.m @@ -753,6 +753,7 @@ function LaunchProtocol(a,b) end ProtocolFolderPath = fullfile(BpodSystem.Path.ProtocolFolder,ProtocolName); ProtocolPath = fullfile(BpodSystem.Path.ProtocolFolder,ProtocolName,[ProtocolName '.m']); +BpodSystem.Path.CurrentProtocol = ProtocolPath; addpath(ProtocolFolderPath); set(BpodSystem.GUIHandles.RunButton, 'cdata', BpodSystem.GUIData.PauseButton, 'TooltipString', 'Press to pause session'); IsOnline = BpodSystem.check4Internet(); diff --git a/Functions/User Functions/SaveSessionVersion.m b/Functions/User Functions/SaveSessionVersion.m new file mode 100644 index 00000000..1ca9283b --- /dev/null +++ b/Functions/User Functions/SaveSessionVersion.m @@ -0,0 +1,125 @@ +function hashOut = SaveSessionVersion(varargin) +%SaveSessionVersion - Save current protocol files and their hashes into SessionData.Info.VersionControl.ProtocolFiles +% SaveSessionVersion(___) +% Using SaveSessionVersion within a protocol file will hash all files in the same directory as the protocol into SessionData.Info.VersionControl.ProtocolFiles. +% Hashes are unique to the file contents, so they can be used to determine if the protocol has changed. +% Hashes are computed using the MD5 algorithm and recorded as hexadecimal strings. +% +% Args: +% excludedExtensions (cell): list of file extensions to exclude from hashing (default: {}) +% addtosessiondata (logical): whether to add the file hashes to the session data (default: true) +% dozip (logical): whether to zip the protocol files (default: true) +% filepaths (cell): list of filepaths to hash (default: [], finds all files) +% protocolpath (char): path to the protocol (default: BpodSystem.Path.CurrentProtocol) +% BpodSystem (struct): BpodSystem struct (default: global BpodSystem) +% verbose (logical): whether to print verbose output (default: false) +% +% Returns (optional): +% fileHashes (struct): struct with fields name and hash (optional) + +% Parse input +p = inputParser(); +p.addParameter('excludedExtensions', {}, @iscell); +p.addParameter('addtosessiondata', true, @islogical); +p.addParameter('dozip', true, @islogical); +p.addParameter('filepaths', [], @iscell); +p.addParameter('protocolpath', [], @ischar); +p.addParameter('BpodSystem', []); % allow overriding of BpodSystem for testing purposes +p.addParameter('verbose', false, @islogical); +p.parse(varargin{:}); + +% Prepare variables +if isempty(p.Results.BpodSystem) + global BpodSystem +else + BpodSystem = p.Results.BpodSystem; +end + +if isempty(p.Results.protocolpath) + protocolpath = BpodSystem.Path.CurrentProtocol; +else + protocolpath = p.Results.protocolpath; + assert(isempty(p.Results.filepaths), 'Cannot specify both protocolpath and filepaths'); +end + +if isfile(protocolpath) + protocolpath = fileparts(protocolpath); +end + +% Determine which files to hash +if isempty(p.Results.filepaths) + if p.Results.verbose + fprintf('Looking for protocol files in %s\n', BpodSystem.Path.CurrentProtocol); + end + filepaths = dir(fullfile(protocolpath, '*.*')); % look at top folders + % remove dots + filepaths = filepaths(~ismember({filepaths.name}, {'.', '..'})); + + % Print list of filenames concatenated + if p.Results.verbose + fprintf('Found %d files in %s:\n\t', length(filepaths), protocolpath); + end +else + filepaths_cell = p.Results.filepaths; + % Turn into struct + filepaths = struct('folder', {}, 'name', {}); + for i = 1:length(filepaths_cell) + [folder, name, ext] = fileparts(filepaths_cell{i}); + filepaths(i).folder = folder; + filepaths(i).name = [name, ext]; + filepaths(i).isdir = isfolder(filepaths_cell{i}); + end +end + + + +% Hash files +fileHashes = struct(); +fileindex = 0; +for i = 1:length(filepaths) + if filepaths(i).isdir + continue + end + [~, ~, ext] = fileparts(filepaths(i).name); + if ismember(ext, p.Results.excludedExtensions) + continue + end + fileindex = fileindex + 1; + fileHashes(i).name = filepaths(i).name; + fileHashes(i).folder = filepaths(i).folder; + fileHashes(i).hash = BpodLib.dataio.HashFile(fullfile(protocolpath, filepaths(i).name)); + if p.Results.verbose + fprintf('%s, ', filepaths(i).name); + end +end +if p.Results.verbose + fprintf('\b\b\n'); + fprintf('Hashed %d files\n', fileindex); +end + +if p.Results.addtosessiondata + if ~isfield(BpodSystem.Data, 'Info') + BpodSystem.Data.Info = struct(); + end + if ~isfield(BpodSystem.Data.Info, 'VersionControl') + BpodSystem.Data.Info.VersionControl = struct(); + end + BpodSystem.Data.Info.VersionControl.ProtocolFiles = fileHashes; +end + +% Zip files +if p.Results.dozip + + filepaths_cell = arrayfun(@(x) fullfile(x.folder, x.name), fileHashes, 'UniformOutput', false); + [savelocation, fname] = fileparts(BpodSystem.Path.CurrentDataFile); + zipfilename = fullfile(savelocation, [fname, '_protocol_files.zip']); + zip(zipfilename, filepaths_cell); + if p.Results.verbose + fprintf('Zipped protocol files to %s\n', zipfilename); + end + +if nargout > 0 + hashOut = fileHashes; +end + +end \ No newline at end of file