From 9ec1391531f7f56adc0bbec11a585fddb409f253 Mon Sep 17 00:00:00 2001 From: javalikescript Date: Sat, 3 Feb 2024 09:16:27 +0100 Subject: [PATCH] Review md5 and sha1 Lua implementation --- jls/util/md/md5-.lua | 236 ++++++++++++++++------------------- jls/util/md/sha1-.lua | 168 ++++++++++++++++++++++++- tests/base/MessageDigest.lua | 39 ++++-- 3 files changed, 302 insertions(+), 141 deletions(-) diff --git a/jls/util/md/md5-.lua b/jls/util/md/md5-.lua index db1ffe4..72716f2 100644 --- a/jls/util/md/md5-.lua +++ b/jls/util/md/md5-.lua @@ -1,49 +1,42 @@ -local char, byte, format, rep, sub = string.char, string.byte, string.format, string.rep, string.sub +-- from https://github.com/kikito/md5.lua -local function bit_not(n) - return ~n -end +--[[ +MIT LICENSE -local function bit_or(m, n) - return m | n -end +Copyright (c) 2013 Enrique García Cota + Adam Baldwin + hanzao + Equi 4 Software -local function bit_and(m, n) - return m & n -end +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: -local function bit_xor(m, n) - return m ~ n -end +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. -local function bit_rshift(n, bits) - return n >> bits -end +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +]] -local function bit_lshift(n, bits) - return n << bits -end +local char, byte, rep, sub = string.char, string.byte, string.rep, string.sub -- convert little-endian 32-bit int to a 4-char string local function lei2str(i) - local f=function (s) return char( bit_and( bit_rshift(i, s), 255)) end - return f(0)..f(8)..f(16)..f(24) -end - --- convert raw string to big-endian int -local function str2bei(s) - local v=0 - for i=1, #s do - v = v * 256 + byte(s, i) - end - return v + return char(i & 255, (i >> 8) & 255, (i >> 16) & 255, (i >> 24) & 255) end -- convert raw string to little-endian int local function str2lei(s) - local v=0 - for i = #s,1,-1 do - v = v*256 + byte(s, i) + local v = 0 + for i = #s, 1, -1 do + v = (v * 256) + byte(s, i) end return v end @@ -59,14 +52,6 @@ local function cut_le_str(s, ...) return r end -local function swap(w) - return str2bei(lei2str(w)) -end - --- An MD5 implementation in Lua --- 10/02/2001 jcw@equi4.com --- see https://github.com/kikito/md5.lua/blob/master/md5.lua - local CONSTS = { 0xd76aa478, 0xe8c7b756, 0x242070db, 0xc1bdceee, 0xf57c0faf, 0x4787c62a, 0xa8304613, 0xfd469501, @@ -87,90 +72,91 @@ local CONSTS = { 0x67452301, 0xefcdab89, 0x98badcfe, 0x10325476 } -local f=function (x,y,z) return bit_or(bit_and(x,y),bit_and(-x-1,z)) end -local g=function (x,y,z) return bit_or(bit_and(x,z),bit_and(y,-z-1)) end -local h=function (x,y,z) return bit_xor(x,bit_xor(y,z)) end -local i=function (x,y,z) return bit_xor(y,bit_or(x,-z-1)) end -local z=function (ff,a,b,c,d,x,s,ac) - a=bit_and(a+ff(b,c,d)+x+ac,0xFFFFFFFF) +local f = function(x, y, z) + return (x & y) | ((-x - 1) & z) +end +local g = function(x, y, z) + return (x & z) | (y & (-z - 1)) +end +local h = function(x, y, z) + return x ~ (y ~ z) +end +local i = function(x, y, z) + return y ~ (x | (-z - 1)) +end +local z = function(ff, a, b, c, d, x, s, ac) + a = (a + ff(b, c, d) + x + ac) & 0xFFFFFFFF -- be *very* careful that left shift does not cause rounding! - return bit_or(bit_lshift(bit_and(a,bit_rshift(0xFFFFFFFF,s)),s),bit_rshift(a,32-s))+b + return (((a & (0xFFFFFFFF >> s)) << s) | (a >> (32 - s))) + b end - -local function transform(A,B,C,D,X) - local a,b,c,d=A,B,C,D - local t=CONSTS - - a=z(f,a,b,c,d,X[ 0], 7,t[ 1]) - d=z(f,d,a,b,c,X[ 1],12,t[ 2]) - c=z(f,c,d,a,b,X[ 2],17,t[ 3]) - b=z(f,b,c,d,a,X[ 3],22,t[ 4]) - a=z(f,a,b,c,d,X[ 4], 7,t[ 5]) - d=z(f,d,a,b,c,X[ 5],12,t[ 6]) - c=z(f,c,d,a,b,X[ 6],17,t[ 7]) - b=z(f,b,c,d,a,X[ 7],22,t[ 8]) - a=z(f,a,b,c,d,X[ 8], 7,t[ 9]) - d=z(f,d,a,b,c,X[ 9],12,t[10]) - c=z(f,c,d,a,b,X[10],17,t[11]) - b=z(f,b,c,d,a,X[11],22,t[12]) - a=z(f,a,b,c,d,X[12], 7,t[13]) - d=z(f,d,a,b,c,X[13],12,t[14]) - c=z(f,c,d,a,b,X[14],17,t[15]) - b=z(f,b,c,d,a,X[15],22,t[16]) - - a=z(g,a,b,c,d,X[ 1], 5,t[17]) - d=z(g,d,a,b,c,X[ 6], 9,t[18]) - c=z(g,c,d,a,b,X[11],14,t[19]) - b=z(g,b,c,d,a,X[ 0],20,t[20]) - a=z(g,a,b,c,d,X[ 5], 5,t[21]) - d=z(g,d,a,b,c,X[10], 9,t[22]) - c=z(g,c,d,a,b,X[15],14,t[23]) - b=z(g,b,c,d,a,X[ 4],20,t[24]) - a=z(g,a,b,c,d,X[ 9], 5,t[25]) - d=z(g,d,a,b,c,X[14], 9,t[26]) - c=z(g,c,d,a,b,X[ 3],14,t[27]) - b=z(g,b,c,d,a,X[ 8],20,t[28]) - a=z(g,a,b,c,d,X[13], 5,t[29]) - d=z(g,d,a,b,c,X[ 2], 9,t[30]) - c=z(g,c,d,a,b,X[ 7],14,t[31]) - b=z(g,b,c,d,a,X[12],20,t[32]) - - a=z(h,a,b,c,d,X[ 5], 4,t[33]) - d=z(h,d,a,b,c,X[ 8],11,t[34]) - c=z(h,c,d,a,b,X[11],16,t[35]) - b=z(h,b,c,d,a,X[14],23,t[36]) - a=z(h,a,b,c,d,X[ 1], 4,t[37]) - d=z(h,d,a,b,c,X[ 4],11,t[38]) - c=z(h,c,d,a,b,X[ 7],16,t[39]) - b=z(h,b,c,d,a,X[10],23,t[40]) - a=z(h,a,b,c,d,X[13], 4,t[41]) - d=z(h,d,a,b,c,X[ 0],11,t[42]) - c=z(h,c,d,a,b,X[ 3],16,t[43]) - b=z(h,b,c,d,a,X[ 6],23,t[44]) - a=z(h,a,b,c,d,X[ 9], 4,t[45]) - d=z(h,d,a,b,c,X[12],11,t[46]) - c=z(h,c,d,a,b,X[15],16,t[47]) - b=z(h,b,c,d,a,X[ 2],23,t[48]) - - a=z(i,a,b,c,d,X[ 0], 6,t[49]) - d=z(i,d,a,b,c,X[ 7],10,t[50]) - c=z(i,c,d,a,b,X[14],15,t[51]) - b=z(i,b,c,d,a,X[ 5],21,t[52]) - a=z(i,a,b,c,d,X[12], 6,t[53]) - d=z(i,d,a,b,c,X[ 3],10,t[54]) - c=z(i,c,d,a,b,X[10],15,t[55]) - b=z(i,b,c,d,a,X[ 1],21,t[56]) - a=z(i,a,b,c,d,X[ 8], 6,t[57]) - d=z(i,d,a,b,c,X[15],10,t[58]) - c=z(i,c,d,a,b,X[ 6],15,t[59]) - b=z(i,b,c,d,a,X[13],21,t[60]) - a=z(i,a,b,c,d,X[ 4], 6,t[61]) - d=z(i,d,a,b,c,X[11],10,t[62]) - c=z(i,c,d,a,b,X[ 2],15,t[63]) - b=z(i,b,c,d,a,X[ 9],21,t[64]) - - return bit_and(A+a,0xFFFFFFFF),bit_and(B+b,0xFFFFFFFF), - bit_and(C+c,0xFFFFFFFF),bit_and(D+d,0xFFFFFFFF) +local function transform(A, B, C, D, X) + local a, b, c, d = A, B, C, D + local t = CONSTS + a = z(f, a, b, c, d, X[0], 7, t[1]) + d = z(f, d, a, b, c, X[1], 12, t[2]) + c = z(f, c, d, a, b, X[2], 17, t[3]) + b = z(f, b, c, d, a, X[3], 22, t[4]) + a = z(f, a, b, c, d, X[4], 7, t[5]) + d = z(f, d, a, b, c, X[5], 12, t[6]) + c = z(f, c, d, a, b, X[6], 17, t[7]) + b = z(f, b, c, d, a, X[7], 22, t[8]) + a = z(f, a, b, c, d, X[8], 7, t[9]) + d = z(f, d, a, b, c, X[9], 12, t[10]) + c = z(f, c, d, a, b, X[10], 17, t[11]) + b = z(f, b, c, d, a, X[11], 22, t[12]) + a = z(f, a, b, c, d, X[12], 7, t[13]) + d = z(f, d, a, b, c, X[13], 12, t[14]) + c = z(f, c, d, a, b, X[14], 17, t[15]) + b = z(f, b, c, d, a, X[15], 22, t[16]) + a = z(g, a, b, c, d, X[1], 5, t[17]) + d = z(g, d, a, b, c, X[6], 9, t[18]) + c = z(g, c, d, a, b, X[11], 14, t[19]) + b = z(g, b, c, d, a, X[0], 20, t[20]) + a = z(g, a, b, c, d, X[5], 5, t[21]) + d = z(g, d, a, b, c, X[10], 9, t[22]) + c = z(g, c, d, a, b, X[15], 14, t[23]) + b = z(g, b, c, d, a, X[4], 20, t[24]) + a = z(g, a, b, c, d, X[9], 5, t[25]) + d = z(g, d, a, b, c, X[14], 9, t[26]) + c = z(g, c, d, a, b, X[3], 14, t[27]) + b = z(g, b, c, d, a, X[8], 20, t[28]) + a = z(g, a, b, c, d, X[13], 5, t[29]) + d = z(g, d, a, b, c, X[2], 9, t[30]) + c = z(g, c, d, a, b, X[7], 14, t[31]) + b = z(g, b, c, d, a, X[12], 20, t[32]) + a = z(h, a, b, c, d, X[5], 4, t[33]) + d = z(h, d, a, b, c, X[8], 11, t[34]) + c = z(h, c, d, a, b, X[11], 16, t[35]) + b = z(h, b, c, d, a, X[14], 23, t[36]) + a = z(h, a, b, c, d, X[1], 4, t[37]) + d = z(h, d, a, b, c, X[4], 11, t[38]) + c = z(h, c, d, a, b, X[7], 16, t[39]) + b = z(h, b, c, d, a, X[10], 23, t[40]) + a = z(h, a, b, c, d, X[13], 4, t[41]) + d = z(h, d, a, b, c, X[0], 11, t[42]) + c = z(h, c, d, a, b, X[3], 16, t[43]) + b = z(h, b, c, d, a, X[6], 23, t[44]) + a = z(h, a, b, c, d, X[9], 4, t[45]) + d = z(h, d, a, b, c, X[12], 11, t[46]) + c = z(h, c, d, a, b, X[15], 16, t[47]) + b = z(h, b, c, d, a, X[2], 23, t[48]) + a = z(i, a, b, c, d, X[0], 6, t[49]) + d = z(i, d, a, b, c, X[7], 10, t[50]) + c = z(i, c, d, a, b, X[14], 15, t[51]) + b = z(i, b, c, d, a, X[5], 21, t[52]) + a = z(i, a, b, c, d, X[12], 6, t[53]) + d = z(i, d, a, b, c, X[3], 10, t[54]) + c = z(i, c, d, a, b, X[10], 15, t[55]) + b = z(i, b, c, d, a, X[1], 21, t[56]) + a = z(i, a, b, c, d, X[8], 6, t[57]) + d = z(i, d, a, b, c, X[15], 10, t[58]) + c = z(i, c, d, a, b, X[6], 15, t[59]) + b = z(i, b, c, d, a, X[13], 21, t[60]) + a = z(i, a, b, c, d, X[4], 6, t[61]) + d = z(i, d, a, b, c, X[11], 10, t[62]) + c = z(i, c, d, a, b, X[2], 15, t[63]) + b = z(i, b, c, d, a, X[9], 21, t[64]) + return (A + a) & 0xFFFFFFFF, (B + b) & 0xFFFFFFFF, (C + c) & 0xFFFFFFFF, (D + d) & 0xFFFFFFFF end return require('jls.lang.class').create('jls.util.MessageDigest', function(md5) @@ -183,7 +169,7 @@ return require('jls.lang.class').create('jls.util.MessageDigest', function(md5) self.pos = self.pos + #s s = self.buf .. s for ii = 1, #s - 63, 64 do - local X = cut_le_str(sub(s,ii,ii+63),4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4) + local X = cut_le_str(sub(s, ii, ii + 63), 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4) assert(#X == 16) X[0] = table.remove(X,1) -- zero based! self.a,self.b,self.c,self.d = transform(self.a,self.b,self.c,self.d,X) @@ -200,7 +186,7 @@ return require('jls.lang.class').create('jls.util.MessageDigest', function(md5) if padLen == 0 then padLen = 64 end - local s = char(128) .. rep(char(0),padLen-1) .. lei2str(bit_and(8*msgLen, 0xFFFFFFFF)) .. lei2str(math.floor(msgLen/0x20000000)) + local s = char(128) .. rep(char(0),padLen-1) .. lei2str((8*msgLen) & 0xFFFFFFFF) .. lei2str(math.floor(msgLen/0x20000000)) self:update(s) assert(self.pos % 64 == 0) diff --git a/jls/util/md/sha1-.lua b/jls/util/md/sha1-.lua index b2ee96b..6daaf48 100644 --- a/jls/util/md/sha1-.lua +++ b/jls/util/md/sha1-.lua @@ -1,4 +1,166 @@ -local sha1Lib = require('sha1') +-- from https://github.com/mpeterv/sha1 + +--[[ +MIT LICENSE + +Copyright (c) 2013 Enrique García Cota, Eike Decker, Jeffrey Friedl +Copyright (c) 2018 Peter Melnichenko + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +]] + +-- Merges four bytes into a uint32 number. +local function bytes_to_uint32(a, b, c, d) + return a * 0x1000000 + b * 0x10000 + c * 0x100 + d +end + +-- Splits a uint32 number into four bytes. +local function uint32_to_bytes(a) + local a4 = a % 256 + a = (a - a4) / 256 + local a3 = a % 256 + a = (a - a3) / 256 + local a2 = a % 256 + local a1 = (a - a2) / 256 + return a1, a2, a3, a4 +end + +local function uint32_lrot(a, bits) + return ((a << bits) & 0xFFFFFFFF) | (a >> (32 - bits)) +end + +local function uint32_xor_3(a, b, c) + return a ~ b ~ c +end + +local function uint32_xor_4(a, b, c, d) + return a ~ b ~ c ~ d +end + +local function uint32_ternary(a, b, c) + -- c ~ (a & (b ~ c)) has less bitwise operations than (a & b) | (~a & c). + return c ~ (a & (b ~ c)) +end + +local function uint32_majority(a, b, c) + -- (a & (b | c)) | (b & c) has less bitwise operations than (a & b) | (a & c) | (b & c). + return (a & (b | c)) | (b & c) +end + +local sbyte = string.byte +local schar = string.char +local srep = string.rep + +-- convert big-endian 32-bit int to a 4-char string +local function bei2str(i) + return schar((i >> 24) & 255, (i >> 16) & 255, (i >> 8) & 255, i & 255) +end + +-- Calculates SHA1 for a string, returns it encoded as 40 hexadecimal digits. +local function digest(str) + -- Input preprocessing. + -- First, append a `1` bit and seven `0` bits. + local first_append = schar(0x80) + + -- Next, append some zero bytes to make the length of the final message a multiple of 64. + -- Eight more bytes will be added next. + local non_zero_message_bytes = #str + 1 + 8 + local second_append = srep(schar(0), -non_zero_message_bytes % 64) + + -- Finally, append the length of the original message in bits as a 64-bit number. + -- Assume that it fits into the lower 32 bits. + local third_append = schar(0, 0, 0, 0, uint32_to_bytes(#str * 8)) + + str = str .. first_append .. second_append .. third_append + assert(#str % 64 == 0) + + -- Initialize hash value. + local h0 = 0x67452301 + local h1 = 0xEFCDAB89 + local h2 = 0x98BADCFE + local h3 = 0x10325476 + local h4 = 0xC3D2E1F0 + + local w = {} + + -- Process the input in successive 64-byte chunks. + for chunk_start = 1, #str, 64 do + -- Load the chunk into W[0..15] as uint32 numbers. + local uint32_start = chunk_start + + for i = 0, 15 do + w[i] = bytes_to_uint32(sbyte(str, uint32_start, uint32_start + 3)) + uint32_start = uint32_start + 4 + end + + -- Extend the input vector. + for i = 16, 79 do + w[i] = uint32_lrot(uint32_xor_4(w[i - 3], w[i - 8], w[i - 14], w[i - 16]), 1) + end + + -- Initialize hash value for this chunk. + local a = h0 + local b = h1 + local c = h2 + local d = h3 + local e = h4 + + -- Main loop. + for i = 0, 79 do + local f + local k + + if i <= 19 then + f = uint32_ternary(b, c, d) + k = 0x5A827999 + elseif i <= 39 then + f = uint32_xor_3(b, c, d) + k = 0x6ED9EBA1 + elseif i <= 59 then + f = uint32_majority(b, c, d) + k = 0x8F1BBCDC + else + f = uint32_xor_3(b, c, d) + k = 0xCA62C1D6 + end + + local temp = (uint32_lrot(a, 5) + f + e + k + w[i]) % 4294967296 + e = d + d = c + c = uint32_lrot(b, 30) + b = a + a = temp + end + + -- Add this chunk's hash to result so far. + h0 = (h0 + a) % 4294967296 + h1 = (h1 + b) % 4294967296 + h2 = (h2 + c) % 4294967296 + h3 = (h3 + d) % 4294967296 + h4 = (h4 + e) % 4294967296 + end + + return bei2str(h0)..bei2str(h1)..bei2str(h2)..bei2str(h3)..bei2str(h4) +end + + local StringBuffer = require('jls.lang.StringBuffer') return require('jls.lang.class').create('jls.util.MessageDigest', function(sha1) @@ -8,12 +170,12 @@ return require('jls.lang.class').create('jls.util.MessageDigest', function(sha1) end function sha1:update(m) - self.buffer:append(m) + self.buffer:append(m) -- FIXME real update return self end function sha1:digest() - return (sha1Lib.binary(self.buffer:toString())) + return digest(self.buffer:toString()) end function sha1:reset() diff --git a/tests/base/MessageDigest.lua b/tests/base/MessageDigest.lua index bdfb5d2..075ce91 100644 --- a/tests/base/MessageDigest.lua +++ b/tests/base/MessageDigest.lua @@ -15,13 +15,24 @@ local function assertHexEquals(result, expected) lu.assertEquals(result, expected, string.format('expected: 0x%X, actual: 0x%X', expected, result)) end +local function onAlgAndMod(alg, mod, fn) + local Md = MessageDigest.getMessageDigest(alg) + fn(Md:new()) + local Mdp = require(mod) + if Md ~= Mdp then + --print('also testing pure Lua module '..mod) + fn(Mdp:new()) + end +end + function Test_md5_digest() - local md = MessageDigest.getInstance('MD5') - lu.assertEquals(hex.encode(md:update(''):digest()), 'd41d8cd98f00b204e9800998ecf8427e') - md:reset():update('The quick brown fox jumps over the lazy dog') - lu.assertEquals(hex.encode(md:digest()), '9e107d9d372bb6826bd81d3542a419d6') - md:reset():update('The quick brown fox jumps over the lazy dog.') - lu.assertEquals(hex.encode(md:digest()), 'e4d909c290d0fb1ca068ffaddf22cbd0') + onAlgAndMod('MD5', 'jls.util.md.md5-', function(md) + lu.assertEquals(hex.encode(md:update(''):digest()), 'd41d8cd98f00b204e9800998ecf8427e') + md:reset():update('The quick brown fox jumps over the lazy dog') + lu.assertEquals(hex.encode(md:digest()), '9e107d9d372bb6826bd81d3542a419d6') + md:reset():update('The quick brown fox jumps over the lazy dog.') + lu.assertEquals(hex.encode(md:digest()), 'e4d909c290d0fb1ca068ffaddf22cbd0') + end) end function Test_md5_updates() @@ -32,16 +43,18 @@ function Test_md5_updates() end function Test_sha1_digest() - local md = MessageDigest.getInstance('SHA-1') - md:update('The quick brown fox jumps over the lazy dog') - lu.assertEquals(hex.encode(md:digest()), '2fd4e1c67a2d28fced849ee1bb76e7391b93eb12') + onAlgAndMod('SHA-1', 'jls.util.md.sha1-', function(md) + md:update('The quick brown fox jumps over the lazy dog') + lu.assertEquals(hex.encode(md:digest()), '2fd4e1c67a2d28fced849ee1bb76e7391b93eb12') + end) end function Test_Crc32_updates() - local md = MessageDigest.getInstance('CRC32') - md:update('The quick brown fox') - md:update(' jumps over the lazy dog') - assertHexEquals(md:digest(), 0x414FA339) + onAlgAndMod('CRC32', 'jls.util.md.crc32-', function(md) + md:update('The quick brown fox') + md:update(' jumps over the lazy dog') + assertHexEquals(md:digest(), 0x414FA339) + end) end function Test_Crc32_digest()