diff --git a/Asm/x86/7zAsm.asm b/Asm/x86/7zAsm.asm index 19c40da..8910d16 100644 --- a/Asm/x86/7zAsm.asm +++ b/Asm/x86/7zAsm.asm @@ -1,5 +1,5 @@ ; 7zAsm.asm -- ASM macros -; 2022-05-16 : Igor Pavlov : Public domain +; 2023-12-08 : Igor Pavlov : Public domain ; UASM can require these changes @@ -43,7 +43,7 @@ else endif endif -OPTION PROLOGUE:NONE +OPTION PROLOGUE:NONE OPTION EPILOGUE:NONE MY_ASM_START macro @@ -121,10 +121,29 @@ endif x2_H equ DH x3_H equ BH +; r0_L equ AL +; r1_L equ CL +; r2_L equ DL +; r3_L equ BL + +; r0_H equ AH +; r1_H equ CH +; r2_H equ DH +; r3_H equ BH + + ifdef x64 x5_L equ BPL x6_L equ SIL x7_L equ DIL + x8_L equ r8b + x9_L equ r9b + x10_L equ r10b + x11_L equ r11b + x12_L equ r12b + x13_L equ r13b + x14_L equ r14b + x15_L equ r15b r0 equ RAX r1 equ RCX @@ -153,6 +172,22 @@ else r7 equ x7 endif + x0_R equ r0 + x1_R equ r1 + x2_R equ r2 + x3_R equ r3 + x4_R equ r4 + x5_R equ r5 + x6_R equ r6 + x7_R equ r7 + x8_R equ r8 + x9_R equ r9 + x10_R equ r10 + x11_R equ r11 + x12_R equ r12 + x13_R equ r13 + x14_R equ r14 + x15_R equ r15 ifdef x64 ifdef ABI_LINUX @@ -200,6 +235,14 @@ REG_ABI_PARAM_0 equ REG_PARAM_0 REG_ABI_PARAM_1_x equ REG_PARAM_1_x REG_ABI_PARAM_1 equ REG_PARAM_1 +MY_PUSH_PRESERVED_ABI_REGS_UP_TO_INCLUDING_R11 macro + MY_PUSH_4_REGS +endm + +MY_POP_PRESERVED_ABI_REGS_UP_TO_INCLUDING_R11 macro + MY_POP_4_REGS +endm + else ; x64 @@ -261,12 +304,25 @@ endm endif ; IS_LINUX -MY_PUSH_PRESERVED_ABI_REGS macro +MY_PUSH_PRESERVED_ABI_REGS_UP_TO_INCLUDING_R11 macro if (IS_LINUX gt 0) MY_PUSH_2_REGS else MY_PUSH_4_REGS endif +endm + +MY_POP_PRESERVED_ABI_REGS_UP_TO_INCLUDING_R11 macro + if (IS_LINUX gt 0) + MY_POP_2_REGS + else + MY_POP_4_REGS + endif +endm + + +MY_PUSH_PRESERVED_ABI_REGS macro + MY_PUSH_PRESERVED_ABI_REGS_UP_TO_INCLUDING_R11 push r12 push r13 push r14 @@ -279,11 +335,7 @@ MY_POP_PRESERVED_ABI_REGS macro pop r14 pop r13 pop r12 - if (IS_LINUX gt 0) - MY_POP_2_REGS - else - MY_POP_4_REGS - endif + MY_POP_PRESERVED_ABI_REGS_UP_TO_INCLUDING_R11 endm endif ; x64 diff --git a/Asm/x86/7zCrcOpt.asm b/Asm/x86/7zCrcOpt.asm index 0fee206..c5de808 100644 --- a/Asm/x86/7zCrcOpt.asm +++ b/Asm/x86/7zCrcOpt.asm @@ -1,180 +1,258 @@ ; 7zCrcOpt.asm -- CRC32 calculation : optimized version -; 2021-02-07 : Igor Pavlov : Public domain +; 2023-12-08 : Igor Pavlov : Public domain include 7zAsm.asm MY_ASM_START -rD equ r2 -rN equ r7 -rT equ r5 +NUM_WORDS equ 3 +UNROLL_CNT equ 2 -ifdef x64 - num_VAR equ r8 - table_VAR equ r9 -else - if (IS_CDECL gt 0) - crc_OFFS equ (REG_SIZE * 5) - data_OFFS equ (REG_SIZE + crc_OFFS) - size_OFFS equ (REG_SIZE + data_OFFS) - else - size_OFFS equ (REG_SIZE * 5) - endif - table_OFFS equ (REG_SIZE + size_OFFS) - num_VAR equ [r4 + size_OFFS] - table_VAR equ [r4 + table_OFFS] +if (NUM_WORDS lt 1) or (NUM_WORDS gt 64) +.err +endif +if (UNROLL_CNT lt 1) +.err endif -SRCDAT equ rD + rN * 1 + 4 * +rD equ r2 +rD_x equ x2 +rN equ r7 +rT equ r5 + +ifndef x64 + if (IS_CDECL gt 0) + crc_OFFS equ (REG_SIZE * 5) + data_OFFS equ (REG_SIZE + crc_OFFS) + size_OFFS equ (REG_SIZE + data_OFFS) + else + size_OFFS equ (REG_SIZE * 5) + endif + table_OFFS equ (REG_SIZE + size_OFFS) +endif + +; rN + rD is same speed as rD, but we reduce one instruction in loop +SRCDAT_1 equ rN + rD * 1 + 1 * +SRCDAT_4 equ rN + rD * 1 + 4 * CRC macro op:req, dest:req, src:req, t:req - op dest, DWORD PTR [rT + src * 4 + 0400h * t] + op dest, dword ptr [rT + @CatStr(src, _R) * 4 + 0400h * (t)] endm CRC_XOR macro dest:req, src:req, t:req - CRC xor, dest, src, t + CRC xor, dest, src, t endm CRC_MOV macro dest:req, src:req, t:req - CRC mov, dest, src, t + CRC mov, dest, src, t +endm + +MOVZXLO macro dest:req, src:req + movzx dest, @CatStr(src, _L) +endm + +MOVZXHI macro dest:req, src:req + movzx dest, @CatStr(src, _H) endm +; movzx x0, x0_L - is slow in some cpus (ivb), if same register for src and dest +; movzx x3, x0_L sometimes is 0 cycles latency (not always) +; movzx x3, x0_L sometimes is 0.5 cycles latency +; movzx x3, x0_H is 2 cycles latency in some cpus + CRC1b macro - movzx x6, BYTE PTR [rD] - inc rD - movzx x3, x0_L - xor x6, x3 - shr x0, 8 - CRC xor, x0, r6, 0 - dec rN + movzx x6, byte ptr [rD] + MOVZXLO x3, x0 + inc rD + shr x0, 8 + xor x6, x3 + CRC_XOR x0, x6, 0 + dec rN +endm + +LOAD_1 macro dest:req, t:req, iter:req, index:req + movzx dest, byte ptr [SRCDAT_1 (4 * (NUM_WORDS - 1 - t + iter * NUM_WORDS) + index)] +endm + +LOAD_2 macro dest:req, t:req, iter:req, index:req + movzx dest, word ptr [SRCDAT_1 (4 * (NUM_WORDS - 1 - t + iter * NUM_WORDS) + index)] +endm + +CRC_QUAD macro nn, t:req, iter:req +ifdef x64 + ; paired memory loads give 1-3% speed gain, but it uses more registers + LOAD_2 x3, t, iter, 0 + LOAD_2 x9, t, iter, 2 + MOVZXLO x6, x3 + shr x3, 8 + CRC_XOR nn, x6, t * 4 + 3 + MOVZXLO x6, x9 + shr x9, 8 + CRC_XOR nn, x3, t * 4 + 2 + CRC_XOR nn, x6, t * 4 + 1 + CRC_XOR nn, x9, t * 4 + 0 +elseif 0 + LOAD_2 x3, t, iter, 0 + MOVZXLO x6, x3 + shr x3, 8 + CRC_XOR nn, x6, t * 4 + 3 + CRC_XOR nn, x3, t * 4 + 2 + LOAD_2 x3, t, iter, 2 + MOVZXLO x6, x3 + shr x3, 8 + CRC_XOR nn, x6, t * 4 + 1 + CRC_XOR nn, x3, t * 4 + 0 +elseif 0 + LOAD_1 x3, t, iter, 0 + LOAD_1 x6, t, iter, 1 + CRC_XOR nn, x3, t * 4 + 3 + CRC_XOR nn, x6, t * 4 + 2 + LOAD_1 x3, t, iter, 2 + LOAD_1 x6, t, iter, 3 + CRC_XOR nn, x3, t * 4 + 1 + CRC_XOR nn, x6, t * 4 + 0 +else + ; 32-bit load is better if there is only one read port (core2) + ; but that code can be slower if there are 2 read ports (snb) + mov x3, dword ptr [SRCDAT_1 (4 * (NUM_WORDS - 1 - t + iter * NUM_WORDS) + 0)] + MOVZXLO x6, x3 + CRC_XOR nn, x6, t * 4 + 3 + MOVZXHI x6, x3 + shr x3, 16 + CRC_XOR nn, x6, t * 4 + 2 + MOVZXLO x6, x3 + shr x3, 8 + CRC_XOR nn, x6, t * 4 + 1 + CRC_XOR nn, x3, t * 4 + 0 +endif endm -MY_PROLOG macro crc_end:req +LAST equ (4 * (NUM_WORDS - 1)) + +CRC_ITER macro qq, nn, iter + mov nn, [SRCDAT_4 (NUM_WORDS * (1 + iter))] + + i = 0 + rept NUM_WORDS - 1 + CRC_QUAD nn, i, iter + i = i + 1 + endm + + MOVZXLO x6, qq + mov x3, qq + shr x3, 24 + CRC_XOR nn, x6, LAST + 3 + CRC_XOR nn, x3, LAST + 0 + ror qq, 16 + MOVZXLO x6, qq + shr qq, 24 + CRC_XOR nn, x6, LAST + 1 +if ((UNROLL_CNT and 1) eq 1) and (iter eq (UNROLL_CNT - 1)) + CRC_MOV qq, qq, LAST + 2 + xor qq, nn +else + CRC_XOR nn, qq, LAST + 2 +endif +endm + + +; + 4 for prefetching next 4-bytes after current iteration +NUM_BYTES_LIMIT equ (NUM_WORDS * 4 * UNROLL_CNT + 4) +ALIGN_MASK equ 3 + + +; MY_PROC @CatStr(CrcUpdateT, 12), 4 +MY_PROC @CatStr(CrcUpdateT, %(NUM_WORDS * 4)), 4 + MY_PUSH_PRESERVED_ABI_REGS_UP_TO_INCLUDING_R11 ifdef x64 + mov x0, REG_ABI_PARAM_0_x ; x0 = x1(win) / x7(linux) + mov rT, REG_ABI_PARAM_3 ; r5 = r9(win) / x1(linux) + mov rN, REG_ABI_PARAM_2 ; r7 = r8(win) / r2(linux) + ; mov rD, REG_ABI_PARAM_1 ; r2 = r2(win) if (IS_LINUX gt 0) - MY_PUSH_2_REGS - mov x0, REG_ABI_PARAM_0_x ; x0 = x7 - mov rT, REG_ABI_PARAM_3 ; r5 = r1 - mov rN, REG_ABI_PARAM_2 ; r7 = r2 mov rD, REG_ABI_PARAM_1 ; r2 = r6 - else - MY_PUSH_4_REGS - mov x0, REG_ABI_PARAM_0_x ; x0 = x1 - mov rT, REG_ABI_PARAM_3 ; r5 = r9 - mov rN, REG_ABI_PARAM_2 ; r7 = r8 - ; mov rD, REG_ABI_PARAM_1 ; r2 = r2 endif else - MY_PUSH_4_REGS if (IS_CDECL gt 0) mov x0, [r4 + crc_OFFS] mov rD, [r4 + data_OFFS] else mov x0, REG_ABI_PARAM_0_x endif - mov rN, num_VAR - mov rT, table_VAR + mov rN, [r4 + size_OFFS] + mov rT, [r4 + table_OFFS] endif - test rN, rN - jz crc_end - @@: - test rD, 7 - jz @F - CRC1b - jnz @B - @@: - cmp rN, 16 - jb crc_end - add rN, rD - mov num_VAR, rN - sub rN, 8 - and rN, NOT 7 - sub rD, rN - xor x0, [SRCDAT 0] -endm + cmp rN, NUM_BYTES_LIMIT + ALIGN_MASK + jb crc_end +@@: + test rD_x, ALIGN_MASK ; test rD, ALIGN_MASK + jz @F + CRC1b + jmp @B +@@: + xor x0, dword ptr [rD] + lea rN, [rD + rN * 1 - (NUM_BYTES_LIMIT - 1)] + sub rD, rN -MY_EPILOG macro crc_end:req - xor x0, [SRCDAT 0] - mov rD, rN - mov rN, num_VAR - sub rN, rD - crc_end: - test rN, rN - jz @F - CRC1b - jmp crc_end - @@: - if (IS_X64 gt 0) and (IS_LINUX gt 0) - MY_POP_2_REGS - else - MY_POP_4_REGS - endif +align 16 +@@: +unr_index = 0 +while unr_index lt UNROLL_CNT + if (unr_index and 1) eq 0 + CRC_ITER x0, x1, unr_index + else + CRC_ITER x1, x0, unr_index + endif + unr_index = unr_index + 1 endm -MY_PROC CrcUpdateT8, 4 - MY_PROLOG crc_end_8 - mov x1, [SRCDAT 1] - align 16 - main_loop_8: - mov x6, [SRCDAT 2] - movzx x3, x1_L - CRC_XOR x6, r3, 3 - movzx x3, x1_H - CRC_XOR x6, r3, 2 - shr x1, 16 - movzx x3, x1_L - movzx x1, x1_H - CRC_XOR x6, r3, 1 - movzx x3, x0_L - CRC_XOR x6, r1, 0 - - mov x1, [SRCDAT 3] - CRC_XOR x6, r3, 7 - movzx x3, x0_H - shr x0, 16 - CRC_XOR x6, r3, 6 - movzx x3, x0_L - CRC_XOR x6, r3, 5 - movzx x3, x0_H - CRC_MOV x0, r3, 4 - xor x0, x6 - add rD, 8 - jnz main_loop_8 - - MY_EPILOG crc_end_8 -MY_ENDP + add rD, NUM_WORDS * 4 * UNROLL_CNT + jnc @B + +if 0 + ; byte verson + add rD, rN + xor x0, dword ptr [rD] + add rN, NUM_BYTES_LIMIT - 1 +else + ; 4-byte version + add rN, 4 * NUM_WORDS * UNROLL_CNT + sub rD, 4 * NUM_WORDS * UNROLL_CNT +@@: + MOVZXLO x3, x0 + MOVZXHI x1, x0 + shr x0, 16 + MOVZXLO x6, x0 + shr x0, 8 + CRC_MOV x0, x0, 0 + CRC_XOR x0, x3, 3 + CRC_XOR x0, x1, 2 + CRC_XOR x0, x6, 1 + + add rD, 4 +if (NUM_WORDS * UNROLL_CNT) ne 1 + jc @F + xor x0, [SRCDAT_4 0] + jmp @B +@@: +endif + add rD, rN + add rN, 4 - 1 + +endif + + sub rN, rD +crc_end: + test rN, rN + jz func_end +@@: + CRC1b + jnz @B -MY_PROC CrcUpdateT4, 4 - MY_PROLOG crc_end_4 - align 16 - main_loop_4: - movzx x1, x0_L - movzx x3, x0_H - shr x0, 16 - movzx x6, x0_H - and x0, 0FFh - CRC_MOV x1, r1, 3 - xor x1, [SRCDAT 1] - CRC_XOR x1, r3, 2 - CRC_XOR x1, r6, 0 - CRC_XOR x1, r0, 1 - - movzx x0, x1_L - movzx x3, x1_H - shr x1, 16 - movzx x6, x1_H - and x1, 0FFh - CRC_MOV x0, r0, 3 - xor x0, [SRCDAT 2] - CRC_XOR x0, r3, 2 - CRC_XOR x0, r6, 0 - CRC_XOR x0, r1, 1 - add rD, 8 - jnz main_loop_4 - - MY_EPILOG crc_end_4 +func_end: + MY_POP_PRESERVED_ABI_REGS_UP_TO_INCLUDING_R11 MY_ENDP end diff --git a/Asm/x86/XzCrc64Opt.asm b/Asm/x86/XzCrc64Opt.asm index ad22cc2..283424c 100644 --- a/Asm/x86/XzCrc64Opt.asm +++ b/Asm/x86/XzCrc64Opt.asm @@ -1,113 +1,231 @@ ; XzCrc64Opt.asm -- CRC64 calculation : optimized version -; 2021-02-06 : Igor Pavlov : Public domain +; 2023-12-08 : Igor Pavlov : Public domain include 7zAsm.asm MY_ASM_START +NUM_WORDS equ 3 + +if (NUM_WORDS lt 1) or (NUM_WORDS gt 64) +.err +endif + +NUM_SKIP_BYTES equ ((NUM_WORDS - 2) * 4) + + +MOVZXLO macro dest:req, src:req + movzx dest, @CatStr(src, _L) +endm + +MOVZXHI macro dest:req, src:req + movzx dest, @CatStr(src, _H) +endm + + ifdef x64 -rD equ r9 +rD equ r11 rN equ r10 -rT equ r5 -num_VAR equ r8 - -SRCDAT4 equ dword ptr [rD + rN * 1] +rT equ r9 + +CRC_OP macro op:req, dest:req, src:req, t:req + op dest, QWORD PTR [rT + @CatStr(src, _R) * 8 + 0800h * (t)] +endm CRC_XOR macro dest:req, src:req, t:req - xor dest, QWORD PTR [rT + src * 8 + 0800h * t] + CRC_OP xor, dest, src, t +endm + +CRC_MOV macro dest:req, src:req, t:req + CRC_OP mov, dest, src, t endm CRC1b macro - movzx x6, BYTE PTR [rD] - inc rD - movzx x3, x0_L - xor x6, x3 - shr r0, 8 - CRC_XOR r0, r6, 0 - dec rN -endm - -MY_PROLOG macro crc_end:req - ifdef ABI_LINUX - MY_PUSH_2_REGS - else - MY_PUSH_4_REGS - endif - mov r0, REG_ABI_PARAM_0 - mov rN, REG_ABI_PARAM_2 - mov rT, REG_ABI_PARAM_3 - mov rD, REG_ABI_PARAM_1 - test rN, rN - jz crc_end - @@: - test rD, 3 - jz @F - CRC1b - jnz @B - @@: - cmp rN, 8 - jb crc_end - add rN, rD - mov num_VAR, rN - sub rN, 4 - and rN, NOT 3 - sub rD, rN - mov x1, SRCDAT4 - xor r0, r1 - add rN, 4 -endm - -MY_EPILOG macro crc_end:req - sub rN, 4 - mov x1, SRCDAT4 - xor r0, r1 - mov rD, rN - mov rN, num_VAR - sub rN, rD - crc_end: - test rN, rN - jz @F - CRC1b - jmp crc_end - @@: - ifdef ABI_LINUX - MY_POP_2_REGS - else - MY_POP_4_REGS - endif + movzx x6, BYTE PTR [rD] + inc rD + MOVZXLO x3, x0 + xor x6, x3 + shr r0, 8 + CRC_XOR r0, x6, 0 + dec rN endm -MY_PROC XzCrc64UpdateT4, 4 - MY_PROLOG crc_end_4 - align 16 - main_loop_4: - mov x1, SRCDAT4 - movzx x2, x0_L - movzx x3, x0_H - shr r0, 16 - movzx x6, x0_L - movzx x7, x0_H - shr r0, 16 - CRC_XOR r1, r2, 3 - CRC_XOR r0, r3, 2 - CRC_XOR r1, r6, 1 - CRC_XOR r0, r7, 0 - xor r0, r1 - - add rD, 4 - jnz main_loop_4 - - MY_EPILOG crc_end_4 + +; ALIGN_MASK is 3 or 7 bytes alignment: +ALIGN_MASK equ (7 - (NUM_WORDS and 1) * 4) + +if NUM_WORDS eq 1 + +src_rN_offset equ 4 +; + 4 for prefetching next 4-bytes after current iteration +NUM_BYTES_LIMIT equ (NUM_WORDS * 4 + 4) +SRCDAT4 equ DWORD PTR [rN + rD * 1] + +XOR_NEXT macro + mov x1, [rD] + xor r0, r1 +endm + +else ; NUM_WORDS > 1 + +src_rN_offset equ 8 +; + 8 for prefetching next 8-bytes after current iteration +NUM_BYTES_LIMIT equ (NUM_WORDS * 4 + 8) + +XOR_NEXT macro + xor r0, QWORD PTR [rD] ; 64-bit read, can be unaligned +endm + +; 32-bit or 64-bit +LOAD_SRC_MULT4 macro dest:req, word_index:req + mov dest, [rN + rD * 1 + 4 * (word_index) - src_rN_offset]; +endm + +endif + + + +MY_PROC @CatStr(XzCrc64UpdateT, %(NUM_WORDS * 4)), 4 + MY_PUSH_PRESERVED_ABI_REGS_UP_TO_INCLUDING_R11 + + mov r0, REG_ABI_PARAM_0 ; r0 <- r1 / r7 + mov rD, REG_ABI_PARAM_1 ; r11 <- r2 / r6 + mov rN, REG_ABI_PARAM_2 ; r10 <- r8 / r2 +if (IS_LINUX gt 0) + mov rT, REG_ABI_PARAM_3 ; r9 <- r9 / r1 +endif + + cmp rN, NUM_BYTES_LIMIT + ALIGN_MASK + jb crc_end +@@: + test rD, ALIGN_MASK + jz @F + CRC1b + jmp @B +@@: + XOR_NEXT + lea rN, [rD + rN * 1 - (NUM_BYTES_LIMIT - 1)] + sub rD, rN + add rN, src_rN_offset + +align 16 +@@: + +if NUM_WORDS eq 1 + + mov x1, x0 + shr x1, 8 + MOVZXLO x3, x1 + MOVZXLO x2, x0 + shr x1, 8 + shr r0, 32 + xor x0, SRCDAT4 + CRC_XOR r0, x2, 3 + CRC_XOR r0, x3, 2 + MOVZXLO x2, x1 + shr x1, 8 + CRC_XOR r0, x2, 1 + CRC_XOR r0, x1, 0 + +else ; NUM_WORDS > 1 + +if NUM_WORDS ne 2 + k = 2 + while k lt NUM_WORDS + + LOAD_SRC_MULT4 x1, k + crc_op1 textequ + + if k eq 2 + if (NUM_WORDS and 1) + LOAD_SRC_MULT4 x7, NUM_WORDS ; aligned 32-bit + LOAD_SRC_MULT4 x6, NUM_WORDS + 1 ; aligned 32-bit + shl r6, 32 + else + LOAD_SRC_MULT4 r6, NUM_WORDS ; aligned 64-bit + crc_op1 textequ + endif + endif + table = 4 * (NUM_WORDS - 1 - k) + MOVZXLO x3, x1 + CRC_OP crc_op1, r7, x3, 3 + table + MOVZXHI x3, x1 + shr x1, 16 + CRC_XOR r6, x3, 2 + table + MOVZXLO x3, x1 + shr x1, 8 + CRC_XOR r7, x3, 1 + table + CRC_XOR r6, x1, 0 + table + k = k + 1 + endm + crc_op2 textequ + +else ; NUM_WORDS == 2 + LOAD_SRC_MULT4 r6, NUM_WORDS ; aligned 64-bit + crc_op2 textequ +endif ; NUM_WORDS == 2 + + MOVZXHI x3, x0 + MOVZXLO x2, x0 + mov r1, r0 + shr r1, 32 + shr x0, 16 + CRC_XOR r6, x2, NUM_SKIP_BYTES + 7 + CRC_OP crc_op2, r7, x3, NUM_SKIP_BYTES + 6 + MOVZXLO x2, x0 + MOVZXHI x5, x1 + MOVZXLO x3, x1 + shr x0, 8 + shr x1, 16 + CRC_XOR r7, x2, NUM_SKIP_BYTES + 5 + CRC_XOR r6, x3, NUM_SKIP_BYTES + 3 + CRC_XOR r7, x0, NUM_SKIP_BYTES + 4 + CRC_XOR r6, x5, NUM_SKIP_BYTES + 2 + MOVZXLO x2, x1 + shr x1, 8 + CRC_XOR r7, x2, NUM_SKIP_BYTES + 1 + CRC_MOV r0, x1, NUM_SKIP_BYTES + 0 + xor r0, r6 + xor r0, r7 + +endif ; NUM_WORDS > 1 + add rD, NUM_WORDS * 4 + jnc @B + + sub rN, src_rN_offset + add rD, rN + XOR_NEXT + add rN, NUM_BYTES_LIMIT - 1 + sub rN, rD + +crc_end: + test rN, rN + jz func_end +@@: + CRC1b + jnz @B +func_end: + MY_POP_PRESERVED_ABI_REGS_UP_TO_INCLUDING_R11 MY_ENDP + + else +; ================================================================== ; x86 (32-bit) -rD equ r1 -rN equ r7 +rD equ r7 +rN equ r1 rT equ r5 +xA equ x6 +xA_R equ r6 + +ifdef x64 + num_VAR equ r8 +else + crc_OFFS equ (REG_SIZE * 5) if (IS_CDECL gt 0) or (IS_LINUX gt 0) @@ -133,107 +251,273 @@ else table_VAR equ [r4 + table_OFFS] num_VAR equ table_VAR endif +endif ; x64 + +SRCDAT4 equ DWORD PTR [rN + rD * 1] -SRCDAT4 equ dword ptr [rD + rN * 1] +CRC_1 macro op:req, dest:req, src:req, t:req, word_index:req + op dest, DWORD PTR [rT + @CatStr(src, _R) * 8 + 0800h * (t) + (word_index) * 4] +endm CRC macro op0:req, op1:req, dest0:req, dest1:req, src:req, t:req - op0 dest0, DWORD PTR [rT + src * 8 + 0800h * t] - op1 dest1, DWORD PTR [rT + src * 8 + 0800h * t + 4] + CRC_1 op0, dest0, src, t, 0 + CRC_1 op1, dest1, src, t, 1 endm CRC_XOR macro dest0:req, dest1:req, src:req, t:req - CRC xor, xor, dest0, dest1, src, t + CRC xor, xor, dest0, dest1, src, t endm CRC1b macro - movzx x6, BYTE PTR [rD] - inc rD - movzx x3, x0_L - xor x6, x3 - shrd r0, r2, 8 - shr r2, 8 - CRC_XOR r0, r2, r6, 0 - dec rN -endm - -MY_PROLOG macro crc_end:req - MY_PUSH_4_REGS - - if (IS_CDECL gt 0) or (IS_LINUX gt 0) - proc_numParams = proc_numParams + 2 ; for ABI_LINUX - mov rN, [r4 + size_OFFS] - mov rD, [r4 + data_OFFS] + movzx xA, BYTE PTR [rD] + inc rD + MOVZXLO x3, x0 + xor xA, x3 + shrd x0, x2, 8 + shr x2, 8 + CRC_XOR x0, x2, xA, 0 + dec rN +endm + + +MY_PROLOG_BASE macro + MY_PUSH_4_REGS +ifdef x64 + mov r0, REG_ABI_PARAM_0 ; r0 <- r1 / r7 + mov rT, REG_ABI_PARAM_3 ; r5 <- r9 / r1 + mov rN, REG_ABI_PARAM_2 ; r1 <- r8 / r2 + mov rD, REG_ABI_PARAM_1 ; r7 <- r2 / r6 + mov r2, r0 + shr r2, 32 + mov x0, x0 +else + if (IS_CDECL gt 0) or (IS_LINUX gt 0) + proc_numParams = proc_numParams + 2 ; for ABI_LINUX + mov rN, [r4 + size_OFFS] + mov rD, [r4 + data_OFFS] + else + mov rD, REG_ABI_PARAM_0 ; r7 <- r1 : (data) + mov rN, REG_ABI_PARAM_1 ; r1 <- r2 : (size) + endif + mov x0, [r4 + crc_OFFS] + mov x2, [r4 + crc_OFFS + 4] + mov rT, table_VAR +endif +endm + + +MY_EPILOG_BASE macro crc_end:req, func_end:req +crc_end: + test rN, rN + jz func_end +@@: + CRC1b + jnz @B +func_end: +ifdef x64 + shl r2, 32 + xor r0, r2 +endif + MY_POP_4_REGS +endm + + +; ALIGN_MASK is 3 or 7 bytes alignment: +ALIGN_MASK equ (7 - (NUM_WORDS and 1) * 4) + +if (NUM_WORDS eq 1) + +NUM_BYTES_LIMIT_T4 equ (NUM_WORDS * 4 + 4) + +MY_PROC @CatStr(XzCrc64UpdateT, %(NUM_WORDS * 4)), 5 + MY_PROLOG_BASE + + cmp rN, NUM_BYTES_LIMIT_T4 + ALIGN_MASK + jb crc_end_4 +@@: + test rD, ALIGN_MASK + jz @F + CRC1b + jmp @B +@@: + xor x0, [rD] + lea rN, [rD + rN * 1 - (NUM_BYTES_LIMIT_T4 - 1)] + sub rD, rN + add rN, 4 + + MOVZXLO xA, x0 +align 16 +@@: + mov x3, SRCDAT4 + xor x3, x2 + shr x0, 8 + CRC xor, mov, x3, x2, xA, 3 + MOVZXLO xA, x0 + shr x0, 8 + ; MOVZXHI xA, x0 + ; shr x0, 16 + CRC_XOR x3, x2, xA, 2 + + MOVZXLO xA, x0 + shr x0, 8 + CRC_XOR x3, x2, xA, 1 + CRC_XOR x3, x2, x0, 0 + MOVZXLO xA, x3 + mov x0, x3 + + add rD, 4 + jnc @B + + sub rN, 4 + add rD, rN + xor x0, [rD] + add rN, NUM_BYTES_LIMIT_T4 - 1 + sub rN, rD + MY_EPILOG_BASE crc_end_4, func_end_4 +MY_ENDP + +else ; NUM_WORDS > 1 + +SHR_X macro x, imm + shr x, imm +endm + + +ITER_1 macro v0, v1, a, off + MOVZXLO xA, a + SHR_X a, 8 + CRC_XOR v0, v1, xA, off +endm + + +ITER_4 macro v0, v1, a, off +if 0 eq 0 + ITER_1 v0, v1, a, off + 3 + ITER_1 v0, v1, a, off + 2 + ITER_1 v0, v1, a, off + 1 + CRC_XOR v0, v1, a, off +elseif 0 eq 0 + MOVZXLO xA, a + CRC_XOR v0, v1, xA, off + 3 + mov xA, a + ror a, 16 ; 32-bit ror + shr xA, 24 + CRC_XOR v0, v1, xA, off + MOVZXLO xA, a + SHR_X a, 24 + CRC_XOR v0, v1, xA, off + 1 + CRC_XOR v0, v1, a, off + 2 +else + ; MOVZXHI provides smaller code, but MOVZX_HI_BYTE is not fast instruction + MOVZXLO xA, a + CRC_XOR v0, v1, xA, off + 3 + MOVZXHI xA, a + SHR_X a, 16 + CRC_XOR v0, v1, xA, off + 2 + MOVZXLO xA, a + SHR_X a, 8 + CRC_XOR v0, v1, xA, off + 1 + CRC_XOR v0, v1, a, off +endif +endm + + + +ITER_1_PAIR macro v0, v1, a0, a1, off + ITER_1 v0, v1, a0, off + 4 + ITER_1 v0, v1, a1, off +endm + +src_rD_offset equ 8 +STEP_SIZE equ (NUM_WORDS * 4) + +ITER_12_NEXT macro op, index, v0, v1 + op v0, DWORD PTR [rD + (index + 1) * STEP_SIZE - src_rD_offset] + op v1, DWORD PTR [rD + (index + 1) * STEP_SIZE + 4 - src_rD_offset] +endm + +ITER_12 macro index, a0, a1, v0, v1 + + if NUM_SKIP_BYTES eq 0 + ITER_12_NEXT mov, index, v0, v1 else - mov rN, r2 + k = 0 + while k lt NUM_SKIP_BYTES + movzx xA, BYTE PTR [rD + (index) * STEP_SIZE + k + 8 - src_rD_offset] + if k eq 0 + CRC mov, mov, v0, v1, xA, NUM_SKIP_BYTES - 1 - k + else + CRC_XOR v0, v1, xA, NUM_SKIP_BYTES - 1 - k + endif + k = k + 1 + endm + ITER_12_NEXT xor, index, v0, v1 endif - mov x0, [r4 + crc_OFFS] - mov x2, [r4 + crc_OFFS + 4] - mov rT, table_VAR - test rN, rN - jz crc_end - @@: - test rD, 3 - jz @F - CRC1b - jnz @B - @@: - cmp rN, 8 - jb crc_end - add rN, rD - - mov num_VAR, rN - - sub rN, 4 - and rN, NOT 3 - sub rD, rN - xor r0, SRCDAT4 - add rN, 4 -endm - -MY_EPILOG macro crc_end:req - sub rN, 4 - xor r0, SRCDAT4 - - mov rD, rN - mov rN, num_VAR - sub rN, rD - crc_end: - test rN, rN - jz @F - CRC1b - jmp crc_end - @@: - MY_POP_4_REGS -endm - -MY_PROC XzCrc64UpdateT4, 5 - MY_PROLOG crc_end_4 - movzx x6, x0_L - align 16 - main_loop_4: - mov r3, SRCDAT4 - xor r3, r2 - - CRC xor, mov, r3, r2, r6, 3 - movzx x6, x0_H - shr r0, 16 - CRC_XOR r3, r2, r6, 2 - - movzx x6, x0_L - movzx x0, x0_H - CRC_XOR r3, r2, r6, 1 - CRC_XOR r3, r2, r0, 0 - movzx x6, x3_L - mov r0, r3 - - add rD, 4 - jnz main_loop_4 - - MY_EPILOG crc_end_4 +if 0 eq 0 + ITER_4 v0, v1, a0, NUM_SKIP_BYTES + 4 + ITER_4 v0, v1, a1, NUM_SKIP_BYTES +else ; interleave version is faster/slower for different processors + ITER_1_PAIR v0, v1, a0, a1, NUM_SKIP_BYTES + 3 + ITER_1_PAIR v0, v1, a0, a1, NUM_SKIP_BYTES + 2 + ITER_1_PAIR v0, v1, a0, a1, NUM_SKIP_BYTES + 1 + CRC_XOR v0, v1, a0, NUM_SKIP_BYTES + 4 + CRC_XOR v0, v1, a1, NUM_SKIP_BYTES +endif +endm + +; we use (UNROLL_CNT > 1) to reduce read ports pressure (num_VAR reads) +UNROLL_CNT equ (2 * 1) +NUM_BYTES_LIMIT equ (STEP_SIZE * UNROLL_CNT + 8) + +MY_PROC @CatStr(XzCrc64UpdateT, %(NUM_WORDS * 4)), 5 + MY_PROLOG_BASE + + cmp rN, NUM_BYTES_LIMIT + ALIGN_MASK + jb crc_end_12 +@@: + test rD, ALIGN_MASK + jz @F + CRC1b + jmp @B +@@: + xor x0, [rD] + xor x2, [rD + 4] + add rD, src_rD_offset + lea rN, [rD + rN * 1 - (NUM_BYTES_LIMIT - 1)] + mov num_VAR, rN + +align 16 +@@: + i = 0 + rept UNROLL_CNT + if (i and 1) eq 0 + ITER_12 i, x0, x2, x1, x3 + else + ITER_12 i, x1, x3, x0, x2 + endif + i = i + 1 + endm + + if (UNROLL_CNT and 1) + mov x0, x1 + mov x2, x3 + endif + add rD, STEP_SIZE * UNROLL_CNT + cmp rD, num_VAR + jb @B + + mov rN, num_VAR + add rN, NUM_BYTES_LIMIT - 1 + sub rN, rD + sub rD, src_rD_offset + xor x0, [rD] + xor x2, [rD + 4] + + MY_EPILOG_BASE crc_end_12, func_end_12 MY_ENDP +endif ; (NUM_WORDS > 1) endif ; ! x64 - end diff --git a/C/7zArcIn.c b/C/7zArcIn.c index 43fa7c2..23f2949 100644 --- a/C/7zArcIn.c +++ b/C/7zArcIn.c @@ -1,5 +1,5 @@ /* 7zArcIn.c -- 7z Input functions -2023-05-11 : Igor Pavlov : Public domain */ +2023-09-07 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -301,7 +301,7 @@ static SRes RememberBitVector(CSzData *sd, UInt32 numItems, const Byte **v) static UInt32 CountDefinedBits(const Byte *bits, UInt32 numItems) { - Byte b = 0; + unsigned b = 0; unsigned m = 0; UInt32 sum = 0; for (; numItems != 0; numItems--) @@ -312,7 +312,7 @@ static UInt32 CountDefinedBits(const Byte *bits, UInt32 numItems) m = 8; } m--; - sum += ((b >> m) & 1); + sum += (UInt32)((b >> m) & 1); } return sum; } diff --git a/C/7zCrc.c b/C/7zCrc.c index c995a8b..6e2db9e 100644 --- a/C/7zCrc.c +++ b/C/7zCrc.c @@ -1,93 +1,96 @@ /* 7zCrc.c -- CRC32 calculation and init -2023-04-02 : Igor Pavlov : Public domain */ +2024-03-01 : Igor Pavlov : Public domain */ #include "Precomp.h" #include "7zCrc.h" #include "CpuArch.h" -#define kCrcPoly 0xEDB88320 +// for debug: +// #define __ARM_FEATURE_CRC32 1 -#ifdef MY_CPU_LE - #define CRC_NUM_TABLES 8 -#else - #define CRC_NUM_TABLES 9 +#ifdef __ARM_FEATURE_CRC32 +// #pragma message("__ARM_FEATURE_CRC32") +#define Z7_CRC_HW_FORCE +#endif - UInt32 Z7_FASTCALL CrcUpdateT1_BeT4(UInt32 v, const void *data, size_t size, const UInt32 *table); - UInt32 Z7_FASTCALL CrcUpdateT1_BeT8(UInt32 v, const void *data, size_t size, const UInt32 *table); +// #define Z7_CRC_DEBUG_BE +#ifdef Z7_CRC_DEBUG_BE +#undef MY_CPU_LE +#define MY_CPU_BE #endif -#ifndef MY_CPU_BE - UInt32 Z7_FASTCALL CrcUpdateT4(UInt32 v, const void *data, size_t size, const UInt32 *table); - UInt32 Z7_FASTCALL CrcUpdateT8(UInt32 v, const void *data, size_t size, const UInt32 *table); +#ifdef Z7_CRC_HW_FORCE + #define Z7_CRC_NUM_TABLES_USE 1 +#else +#ifdef Z7_CRC_NUM_TABLES + #define Z7_CRC_NUM_TABLES_USE Z7_CRC_NUM_TABLES +#else + #define Z7_CRC_NUM_TABLES_USE 12 +#endif #endif -/* -extern -CRC_FUNC g_CrcUpdateT4; -CRC_FUNC g_CrcUpdateT4; -*/ -extern -CRC_FUNC g_CrcUpdateT8; -CRC_FUNC g_CrcUpdateT8; -extern -CRC_FUNC g_CrcUpdateT0_32; -CRC_FUNC g_CrcUpdateT0_32; -extern -CRC_FUNC g_CrcUpdateT0_64; -CRC_FUNC g_CrcUpdateT0_64; -extern -CRC_FUNC g_CrcUpdate; -CRC_FUNC g_CrcUpdate; - -UInt32 g_CrcTable[256 * CRC_NUM_TABLES]; - -UInt32 Z7_FASTCALL CrcUpdate(UInt32 v, const void *data, size_t size) -{ - return g_CrcUpdate(v, data, size, g_CrcTable); -} +#if Z7_CRC_NUM_TABLES_USE < 1 + #error Stop_Compiling_Bad_Z7_CRC_NUM_TABLES +#endif -UInt32 Z7_FASTCALL CrcCalc(const void *data, size_t size) -{ - return g_CrcUpdate(CRC_INIT_VAL, data, size, g_CrcTable) ^ CRC_INIT_VAL; -} +#if defined(MY_CPU_LE) || (Z7_CRC_NUM_TABLES_USE == 1) + #define Z7_CRC_NUM_TABLES_TOTAL Z7_CRC_NUM_TABLES_USE +#else + #define Z7_CRC_NUM_TABLES_TOTAL (Z7_CRC_NUM_TABLES_USE + 1) +#endif -#if CRC_NUM_TABLES < 4 \ - || (CRC_NUM_TABLES == 4 && defined(MY_CPU_BE)) \ +#ifndef Z7_CRC_HW_FORCE + +#if Z7_CRC_NUM_TABLES_USE == 1 \ || (!defined(MY_CPU_LE) && !defined(MY_CPU_BE)) -#define CRC_UPDATE_BYTE_2(crc, b) (table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8)) -UInt32 Z7_FASTCALL CrcUpdateT1(UInt32 v, const void *data, size_t size, const UInt32 *table); -UInt32 Z7_FASTCALL CrcUpdateT1(UInt32 v, const void *data, size_t size, const UInt32 *table) +#define CRC_UPDATE_BYTE_2(crc, b) (table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8)) +#define Z7_CRC_UPDATE_T1_FUNC_NAME CrcUpdateGT1 +static UInt32 Z7_FASTCALL Z7_CRC_UPDATE_T1_FUNC_NAME(UInt32 v, const void *data, size_t size) { + const UInt32 *table = g_CrcTable; const Byte *p = (const Byte *)data; - const Byte *pEnd = p + size; - for (; p != pEnd; p++) + const Byte *lim = p + size; + for (; p != lim; p++) v = CRC_UPDATE_BYTE_2(v, *p); return v; } #endif + +#if Z7_CRC_NUM_TABLES_USE != 1 +#ifndef MY_CPU_BE + #define FUNC_NAME_LE_2(s) CrcUpdateT ## s + #define FUNC_NAME_LE_1(s) FUNC_NAME_LE_2(s) + #define FUNC_NAME_LE FUNC_NAME_LE_1(Z7_CRC_NUM_TABLES_USE) + UInt32 Z7_FASTCALL FUNC_NAME_LE (UInt32 v, const void *data, size_t size, const UInt32 *table); +#endif +#ifndef MY_CPU_LE + #define FUNC_NAME_BE_2(s) CrcUpdateT1_BeT ## s + #define FUNC_NAME_BE_1(s) FUNC_NAME_BE_2(s) + #define FUNC_NAME_BE FUNC_NAME_BE_1(Z7_CRC_NUM_TABLES_USE) + UInt32 Z7_FASTCALL FUNC_NAME_BE (UInt32 v, const void *data, size_t size, const UInt32 *table); +#endif +#endif + +#endif // Z7_CRC_HW_FORCE + /* ---------- hardware CRC ---------- */ #ifdef MY_CPU_LE #if defined(MY_CPU_ARM_OR_ARM64) - // #pragma message("ARM*") - #if defined(_MSC_VER) - #if defined(MY_CPU_ARM64) - #if (_MSC_VER >= 1910) - #ifndef __clang__ - #define USE_ARM64_CRC - #include - #endif - #endif - #endif - #elif (defined(__clang__) && (__clang_major__ >= 3)) \ - || (defined(__GNUC__) && (__GNUC__ > 4)) + #if (defined(__clang__) && (__clang_major__ >= 3)) \ + || defined(__GNUC__) && (__GNUC__ >= 6) && defined(MY_CPU_ARM64) \ + || defined(__GNUC__) && (__GNUC__ >= 8) #if !defined(__ARM_FEATURE_CRC32) +// #pragma message("!defined(__ARM_FEATURE_CRC32)") +Z7_DIAGNOSTIC_IGNORE_BEGIN_RESERVED_MACRO_IDENTIFIER #define __ARM_FEATURE_CRC32 1 +Z7_DIAGNOSTIC_IGNORE_END_RESERVED_MACRO_IDENTIFIER + #define Z7_ARM_FEATURE_CRC32_WAS_SET #if defined(__clang__) #if defined(MY_CPU_ARM64) #define ATTRIB_CRC __attribute__((__target__("crc"))) @@ -96,100 +99,120 @@ UInt32 Z7_FASTCALL CrcUpdateT1(UInt32 v, const void *data, size_t size, const UI #endif #else #if defined(MY_CPU_ARM64) +#if !defined(Z7_GCC_VERSION) || (Z7_GCC_VERSION >= 60000) #define ATTRIB_CRC __attribute__((__target__("+crc"))) +#endif #else +#if !defined(Z7_GCC_VERSION) || (__GNUC__ >= 8) +#if defined(__ARM_FP) && __GNUC__ >= 8 +// for -mfloat-abi=hard: similar to + #define ATTRIB_CRC __attribute__((__target__("arch=armv8-a+crc+simd"))) +#else #define ATTRIB_CRC __attribute__((__target__("arch=armv8-a+crc"))) +#endif +#endif #endif #endif #endif #if defined(__ARM_FEATURE_CRC32) - #define USE_ARM64_CRC + // #pragma message("") +/* +arm_acle.h (GGC): + before Nov 17, 2017: +#ifdef __ARM_FEATURE_CRC32 + + Nov 17, 2017: gcc10.0 (gcc 9.2.0) checked" +#if __ARM_ARCH >= 8 +#pragma GCC target ("arch=armv8-a+crc") + + Aug 22, 2019: GCC 8.4?, 9.2.1, 10.1: +#ifdef __ARM_FEATURE_CRC32 +#ifdef __ARM_FP +#pragma GCC target ("arch=armv8-a+crc+simd") +#else +#pragma GCC target ("arch=armv8-a+crc") +#endif +*/ +#if defined(__ARM_ARCH) && __ARM_ARCH < 8 +#if defined(Z7_GCC_VERSION) && (__GNUC__ == 8) && (Z7_GCC_VERSION < 80400) \ + || defined(Z7_GCC_VERSION) && (__GNUC__ == 9) && (Z7_GCC_VERSION < 90201) \ + || defined(Z7_GCC_VERSION) && (__GNUC__ == 10) && (Z7_GCC_VERSION < 100100) +Z7_DIAGNOSTIC_IGNORE_BEGIN_RESERVED_MACRO_IDENTIFIER +// #pragma message("#define __ARM_ARCH 8") +#undef __ARM_ARCH +#define __ARM_ARCH 8 +Z7_DIAGNOSTIC_IGNORE_END_RESERVED_MACRO_IDENTIFIER +#endif +#endif + #define Z7_CRC_HW_USE #include #endif + #elif defined(_MSC_VER) + #if defined(MY_CPU_ARM64) + #if (_MSC_VER >= 1910) + #ifdef __clang__ + // #define Z7_CRC_HW_USE + // #include + #else + #define Z7_CRC_HW_USE + #include + #endif + #endif + #endif #endif -#else - -// no hardware CRC - -// #define USE_CRC_EMU - -#ifdef USE_CRC_EMU - -#pragma message("ARM64 CRC emulation") - -Z7_FORCE_INLINE -UInt32 __crc32b(UInt32 v, UInt32 data) -{ - const UInt32 *table = g_CrcTable; - v = CRC_UPDATE_BYTE_2(v, (Byte)data); - return v; -} +#else // non-ARM* -Z7_FORCE_INLINE -UInt32 __crc32w(UInt32 v, UInt32 data) -{ - const UInt32 *table = g_CrcTable; - v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8; - v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8; - v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8; - v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8; - return v; -} +// #define Z7_CRC_HW_USE // for debug : we can test HW-branch of code +#ifdef Z7_CRC_HW_USE +#include "7zCrcEmu.h" +#endif -Z7_FORCE_INLINE -UInt32 __crc32d(UInt32 v, UInt64 data) -{ - const UInt32 *table = g_CrcTable; - v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8; - v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8; - v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8; - v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8; - v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8; - v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8; - v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8; - v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8; - return v; -} +#endif // non-ARM* -#endif // USE_CRC_EMU -#endif // defined(MY_CPU_ARM64) && defined(MY_CPU_LE) +#if defined(Z7_CRC_HW_USE) +// #pragma message("USE ARM HW CRC") -#if defined(USE_ARM64_CRC) || defined(USE_CRC_EMU) +#ifdef MY_CPU_64BIT + #define CRC_HW_WORD_TYPE UInt64 + #define CRC_HW_WORD_FUNC __crc32d +#else + #define CRC_HW_WORD_TYPE UInt32 + #define CRC_HW_WORD_FUNC __crc32w +#endif -#define T0_32_UNROLL_BYTES (4 * 4) -#define T0_64_UNROLL_BYTES (4 * 8) +#define CRC_HW_UNROLL_BYTES (sizeof(CRC_HW_WORD_TYPE) * 4) -#ifndef ATTRIB_CRC -#define ATTRIB_CRC +#ifdef ATTRIB_CRC + ATTRIB_CRC #endif -// #pragma message("USE ARM HW CRC") - -ATTRIB_CRC -UInt32 Z7_FASTCALL CrcUpdateT0_32(UInt32 v, const void *data, size_t size, const UInt32 *table); -ATTRIB_CRC -UInt32 Z7_FASTCALL CrcUpdateT0_32(UInt32 v, const void *data, size_t size, const UInt32 *table) +Z7_NO_INLINE +#ifdef Z7_CRC_HW_FORCE + UInt32 Z7_FASTCALL CrcUpdate +#else + static UInt32 Z7_FASTCALL CrcUpdate_HW +#endif + (UInt32 v, const void *data, size_t size) { const Byte *p = (const Byte *)data; - UNUSED_VAR(table); - - for (; size != 0 && ((unsigned)(ptrdiff_t)p & (T0_32_UNROLL_BYTES - 1)) != 0; size--) + for (; size != 0 && ((unsigned)(ptrdiff_t)p & (CRC_HW_UNROLL_BYTES - 1)) != 0; size--) v = __crc32b(v, *p++); - - if (size >= T0_32_UNROLL_BYTES) + if (size >= CRC_HW_UNROLL_BYTES) { const Byte *lim = p + size; - size &= (T0_32_UNROLL_BYTES - 1); + size &= CRC_HW_UNROLL_BYTES - 1; lim -= size; do { - v = __crc32w(v, *(const UInt32 *)(const void *)(p)); - v = __crc32w(v, *(const UInt32 *)(const void *)(p + 4)); p += 2 * 4; - v = __crc32w(v, *(const UInt32 *)(const void *)(p)); - v = __crc32w(v, *(const UInt32 *)(const void *)(p + 4)); p += 2 * 4; + v = CRC_HW_WORD_FUNC(v, *(const CRC_HW_WORD_TYPE *)(const void *)(p)); + v = CRC_HW_WORD_FUNC(v, *(const CRC_HW_WORD_TYPE *)(const void *)(p + sizeof(CRC_HW_WORD_TYPE))); + p += 2 * sizeof(CRC_HW_WORD_TYPE); + v = CRC_HW_WORD_FUNC(v, *(const CRC_HW_WORD_TYPE *)(const void *)(p)); + v = CRC_HW_WORD_FUNC(v, *(const CRC_HW_WORD_TYPE *)(const void *)(p + sizeof(CRC_HW_WORD_TYPE))); + p += 2 * sizeof(CRC_HW_WORD_TYPE); } while (p != lim); } @@ -200,46 +223,86 @@ UInt32 Z7_FASTCALL CrcUpdateT0_32(UInt32 v, const void *data, size_t size, const return v; } -ATTRIB_CRC -UInt32 Z7_FASTCALL CrcUpdateT0_64(UInt32 v, const void *data, size_t size, const UInt32 *table); -ATTRIB_CRC -UInt32 Z7_FASTCALL CrcUpdateT0_64(UInt32 v, const void *data, size_t size, const UInt32 *table) +#ifdef Z7_ARM_FEATURE_CRC32_WAS_SET +Z7_DIAGNOSTIC_IGNORE_BEGIN_RESERVED_MACRO_IDENTIFIER +#undef __ARM_FEATURE_CRC32 +Z7_DIAGNOSTIC_IGNORE_END_RESERVED_MACRO_IDENTIFIER +#undef Z7_ARM_FEATURE_CRC32_WAS_SET +#endif + +#endif // defined(Z7_CRC_HW_USE) +#endif // MY_CPU_LE + + + +#ifndef Z7_CRC_HW_FORCE + +#if defined(Z7_CRC_HW_USE) || defined(Z7_CRC_UPDATE_T1_FUNC_NAME) +/* +typedef UInt32 (Z7_FASTCALL *Z7_CRC_UPDATE_WITH_TABLE_FUNC) + (UInt32 v, const void *data, size_t size, const UInt32 *table); +Z7_CRC_UPDATE_WITH_TABLE_FUNC g_CrcUpdate; +*/ +static unsigned g_Crc_Algo; +#if (!defined(MY_CPU_LE) && !defined(MY_CPU_BE)) +static unsigned g_Crc_Be; +#endif +#endif // defined(Z7_CRC_HW_USE) || defined(Z7_CRC_UPDATE_T1_FUNC_NAME) + + + +Z7_NO_INLINE +#ifdef Z7_CRC_HW_USE + static UInt32 Z7_FASTCALL CrcUpdate_Base +#else + UInt32 Z7_FASTCALL CrcUpdate +#endif + (UInt32 crc, const void *data, size_t size) { - const Byte *p = (const Byte *)data; - UNUSED_VAR(table); +#if Z7_CRC_NUM_TABLES_USE == 1 + return Z7_CRC_UPDATE_T1_FUNC_NAME(crc, data, size); +#else // Z7_CRC_NUM_TABLES_USE != 1 +#ifdef Z7_CRC_UPDATE_T1_FUNC_NAME + if (g_Crc_Algo == 1) + return Z7_CRC_UPDATE_T1_FUNC_NAME(crc, data, size); +#endif - for (; size != 0 && ((unsigned)(ptrdiff_t)p & (T0_64_UNROLL_BYTES - 1)) != 0; size--) - v = __crc32b(v, *p++); +#ifdef MY_CPU_LE + return FUNC_NAME_LE(crc, data, size, g_CrcTable); +#elif defined(MY_CPU_BE) + return FUNC_NAME_BE(crc, data, size, g_CrcTable); +#else + if (g_Crc_Be) + return FUNC_NAME_BE(crc, data, size, g_CrcTable); + else + return FUNC_NAME_LE(crc, data, size, g_CrcTable); +#endif +#endif // Z7_CRC_NUM_TABLES_USE != 1 +} - if (size >= T0_64_UNROLL_BYTES) - { - const Byte *lim = p + size; - size &= (T0_64_UNROLL_BYTES - 1); - lim -= size; - do - { - v = __crc32d(v, *(const UInt64 *)(const void *)(p)); - v = __crc32d(v, *(const UInt64 *)(const void *)(p + 8)); p += 2 * 8; - v = __crc32d(v, *(const UInt64 *)(const void *)(p)); - v = __crc32d(v, *(const UInt64 *)(const void *)(p + 8)); p += 2 * 8; - } - while (p != lim); - } - - for (; size != 0; size--) - v = __crc32b(v, *p++); - return v; +#ifdef Z7_CRC_HW_USE +Z7_NO_INLINE +UInt32 Z7_FASTCALL CrcUpdate(UInt32 crc, const void *data, size_t size) +{ + if (g_Crc_Algo == 0) + return CrcUpdate_HW(crc, data, size); + return CrcUpdate_Base(crc, data, size); } +#endif -#undef T0_32_UNROLL_BYTES -#undef T0_64_UNROLL_BYTES +#endif // !defined(Z7_CRC_HW_FORCE) -#endif // defined(USE_ARM64_CRC) || defined(USE_CRC_EMU) -#endif // MY_CPU_LE + +UInt32 Z7_FASTCALL CrcCalc(const void *data, size_t size) +{ + return CrcUpdate(CRC_INIT_VAL, data, size) ^ CRC_INIT_VAL; +} +MY_ALIGN(64) +UInt32 g_CrcTable[256 * Z7_CRC_NUM_TABLES_TOTAL]; void Z7_FASTCALL CrcGenerateTable(void) @@ -247,94 +310,111 @@ void Z7_FASTCALL CrcGenerateTable(void) UInt32 i; for (i = 0; i < 256; i++) { +#if defined(Z7_CRC_HW_FORCE) + g_CrcTable[i] = __crc32b(i, 0); +#else + #define kCrcPoly 0xEDB88320 UInt32 r = i; unsigned j; for (j = 0; j < 8; j++) r = (r >> 1) ^ (kCrcPoly & ((UInt32)0 - (r & 1))); g_CrcTable[i] = r; +#endif } - for (i = 256; i < 256 * CRC_NUM_TABLES; i++) + for (i = 256; i < 256 * Z7_CRC_NUM_TABLES_USE; i++) { const UInt32 r = g_CrcTable[(size_t)i - 256]; g_CrcTable[i] = g_CrcTable[r & 0xFF] ^ (r >> 8); } - #if CRC_NUM_TABLES < 4 - g_CrcUpdate = CrcUpdateT1; - #elif defined(MY_CPU_LE) - // g_CrcUpdateT4 = CrcUpdateT4; - #if CRC_NUM_TABLES < 8 - g_CrcUpdate = CrcUpdateT4; - #else // CRC_NUM_TABLES >= 8 - g_CrcUpdateT8 = CrcUpdateT8; - /* - #ifdef MY_CPU_X86_OR_AMD64 - if (!CPU_Is_InOrder()) - #endif - */ - g_CrcUpdate = CrcUpdateT8; - #endif - #else +#if !defined(Z7_CRC_HW_FORCE) && \ + (defined(Z7_CRC_HW_USE) || defined(Z7_CRC_UPDATE_T1_FUNC_NAME) || defined(MY_CPU_BE)) + +#if Z7_CRC_NUM_TABLES_USE <= 1 + g_Crc_Algo = 1; +#else // Z7_CRC_NUM_TABLES_USE <= 1 + +#if defined(MY_CPU_LE) + g_Crc_Algo = Z7_CRC_NUM_TABLES_USE; +#else // !defined(MY_CPU_LE) { - #ifndef MY_CPU_BE +#ifndef MY_CPU_BE UInt32 k = 0x01020304; const Byte *p = (const Byte *)&k; if (p[0] == 4 && p[1] == 3) - { - #if CRC_NUM_TABLES < 8 - // g_CrcUpdateT4 = CrcUpdateT4; - g_CrcUpdate = CrcUpdateT4; - #else // CRC_NUM_TABLES >= 8 - g_CrcUpdateT8 = CrcUpdateT8; - g_CrcUpdate = CrcUpdateT8; - #endif - } + g_Crc_Algo = Z7_CRC_NUM_TABLES_USE; else if (p[0] != 1 || p[1] != 2) - g_CrcUpdate = CrcUpdateT1; + g_Crc_Algo = 1; else - #endif // MY_CPU_BE +#endif // MY_CPU_BE { - for (i = 256 * CRC_NUM_TABLES - 1; i >= 256; i--) + for (i = 256 * Z7_CRC_NUM_TABLES_TOTAL - 1; i >= 256; i--) { const UInt32 x = g_CrcTable[(size_t)i - 256]; g_CrcTable[i] = Z7_BSWAP32(x); } - #if CRC_NUM_TABLES <= 4 - g_CrcUpdate = CrcUpdateT1; - #elif CRC_NUM_TABLES <= 8 - // g_CrcUpdateT4 = CrcUpdateT1_BeT4; - g_CrcUpdate = CrcUpdateT1_BeT4; - #else // CRC_NUM_TABLES > 8 - g_CrcUpdateT8 = CrcUpdateT1_BeT8; - g_CrcUpdate = CrcUpdateT1_BeT8; - #endif +#if defined(Z7_CRC_UPDATE_T1_FUNC_NAME) + g_Crc_Algo = Z7_CRC_NUM_TABLES_USE; +#endif +#if (!defined(MY_CPU_LE) && !defined(MY_CPU_BE)) + g_Crc_Be = 1; +#endif } } - #endif // CRC_NUM_TABLES < 4 +#endif // !defined(MY_CPU_LE) - #ifdef MY_CPU_LE - #ifdef USE_ARM64_CRC - if (CPU_IsSupported_CRC32()) - { - g_CrcUpdateT0_32 = CrcUpdateT0_32; - g_CrcUpdateT0_64 = CrcUpdateT0_64; - g_CrcUpdate = - #if defined(MY_CPU_ARM) - CrcUpdateT0_32; - #else - CrcUpdateT0_64; - #endif - } - #endif - - #ifdef USE_CRC_EMU - g_CrcUpdateT0_32 = CrcUpdateT0_32; - g_CrcUpdateT0_64 = CrcUpdateT0_64; - g_CrcUpdate = CrcUpdateT0_64; - #endif +#ifdef MY_CPU_LE +#ifdef Z7_CRC_HW_USE + if (CPU_IsSupported_CRC32()) + g_Crc_Algo = 0; +#endif // Z7_CRC_HW_USE +#endif // MY_CPU_LE + +#endif // Z7_CRC_NUM_TABLES_USE <= 1 +#endif // g_Crc_Algo was declared +} + +Z7_CRC_UPDATE_FUNC z7_GetFunc_CrcUpdate(unsigned algo) +{ + if (algo == 0) + return &CrcUpdate; + +#if defined(Z7_CRC_HW_USE) + if (algo == sizeof(CRC_HW_WORD_TYPE) * 8) + { +#ifdef Z7_CRC_HW_FORCE + return &CrcUpdate; +#else + if (g_Crc_Algo == 0) + return &CrcUpdate_HW; +#endif + } +#endif + +#ifndef Z7_CRC_HW_FORCE + if (algo == Z7_CRC_NUM_TABLES_USE) + return + #ifdef Z7_CRC_HW_USE + &CrcUpdate_Base; + #else + &CrcUpdate; #endif +#endif + + return NULL; } #undef kCrcPoly -#undef CRC64_NUM_TABLES +#undef Z7_CRC_NUM_TABLES_USE +#undef Z7_CRC_NUM_TABLES_TOTAL #undef CRC_UPDATE_BYTE_2 +#undef FUNC_NAME_LE_2 +#undef FUNC_NAME_LE_1 +#undef FUNC_NAME_LE +#undef FUNC_NAME_BE_2 +#undef FUNC_NAME_BE_1 +#undef FUNC_NAME_BE + +#undef CRC_HW_UNROLL_BYTES +#undef CRC_HW_WORD_FUNC +#undef CRC_HW_WORD_TYPE diff --git a/C/7zCrc.h b/C/7zCrc.h index 4afaeae..3e6d408 100644 --- a/C/7zCrc.h +++ b/C/7zCrc.h @@ -1,5 +1,5 @@ /* 7zCrc.h -- CRC32 calculation -2023-04-02 : Igor Pavlov : Public domain */ +2024-01-22 : Igor Pavlov : Public domain */ #ifndef ZIP7_INC_7Z_CRC_H #define ZIP7_INC_7Z_CRC_H @@ -20,7 +20,8 @@ void Z7_FASTCALL CrcGenerateTable(void); UInt32 Z7_FASTCALL CrcUpdate(UInt32 crc, const void *data, size_t size); UInt32 Z7_FASTCALL CrcCalc(const void *data, size_t size); -typedef UInt32 (Z7_FASTCALL *CRC_FUNC)(UInt32 v, const void *data, size_t size, const UInt32 *table); +typedef UInt32 (Z7_FASTCALL *Z7_CRC_UPDATE_FUNC)(UInt32 v, const void *data, size_t size); +Z7_CRC_UPDATE_FUNC z7_GetFunc_CrcUpdate(unsigned algo); EXTERN_C_END diff --git a/C/7zCrcOpt.c b/C/7zCrcOpt.c index 9c64929..9408017 100644 --- a/C/7zCrcOpt.c +++ b/C/7zCrcOpt.c @@ -1,117 +1,199 @@ -/* 7zCrcOpt.c -- CRC32 calculation -2023-04-02 : Igor Pavlov : Public domain */ +/* 7zCrcOpt.c -- CRC32 calculation (optimized functions) +2023-12-07 : Igor Pavlov : Public domain */ #include "Precomp.h" #include "CpuArch.h" +#if !defined(Z7_CRC_NUM_TABLES) || Z7_CRC_NUM_TABLES > 1 + +// for debug only : define Z7_CRC_DEBUG_BE to test big-endian code in little-endian cpu +// #define Z7_CRC_DEBUG_BE +#ifdef Z7_CRC_DEBUG_BE +#undef MY_CPU_LE +#define MY_CPU_BE +#endif + +// the value Z7_CRC_NUM_TABLES_USE must be defined to same value as in 7zCrc.c +#ifdef Z7_CRC_NUM_TABLES +#define Z7_CRC_NUM_TABLES_USE Z7_CRC_NUM_TABLES +#else +#define Z7_CRC_NUM_TABLES_USE 12 +#endif + +#if Z7_CRC_NUM_TABLES_USE % 4 || \ + Z7_CRC_NUM_TABLES_USE < 4 * 1 || \ + Z7_CRC_NUM_TABLES_USE > 4 * 6 + #error Stop_Compiling_Bad_Z7_CRC_NUM_TABLES +#endif + + #ifndef MY_CPU_BE -#define CRC_UPDATE_BYTE_2(crc, b) (table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8)) +#define CRC_UPDATE_BYTE_2(crc, b) (table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8)) -UInt32 Z7_FASTCALL CrcUpdateT4(UInt32 v, const void *data, size_t size, const UInt32 *table); -UInt32 Z7_FASTCALL CrcUpdateT4(UInt32 v, const void *data, size_t size, const UInt32 *table) -{ - const Byte *p = (const Byte *)data; - for (; size > 0 && ((unsigned)(ptrdiff_t)p & 3) != 0; size--, p++) - v = CRC_UPDATE_BYTE_2(v, *p); - for (; size >= 4; size -= 4, p += 4) - { - v ^= *(const UInt32 *)(const void *)p; - v = - (table + 0x300)[((v ) & 0xFF)] - ^ (table + 0x200)[((v >> 8) & 0xFF)] - ^ (table + 0x100)[((v >> 16) & 0xFF)] - ^ (table + 0x000)[((v >> 24))]; - } - for (; size > 0; size--, p++) - v = CRC_UPDATE_BYTE_2(v, *p); - return v; -} +#define Q(n, d) \ + ( (table + ((n) * 4 + 3) * 0x100)[(Byte)(d)] \ + ^ (table + ((n) * 4 + 2) * 0x100)[((d) >> 1 * 8) & 0xFF] \ + ^ (table + ((n) * 4 + 1) * 0x100)[((d) >> 2 * 8) & 0xFF] \ + ^ (table + ((n) * 4 + 0) * 0x100)[((d) >> 3 * 8)] ) + +#define R(a) *((const UInt32 *)(const void *)p + (a)) + +#define CRC_FUNC_PRE_LE2(step) \ +UInt32 Z7_FASTCALL CrcUpdateT ## step (UInt32 v, const void *data, size_t size, const UInt32 *table) -UInt32 Z7_FASTCALL CrcUpdateT8(UInt32 v, const void *data, size_t size, const UInt32 *table); -UInt32 Z7_FASTCALL CrcUpdateT8(UInt32 v, const void *data, size_t size, const UInt32 *table) +#define CRC_FUNC_PRE_LE(step) \ + CRC_FUNC_PRE_LE2(step); \ + CRC_FUNC_PRE_LE2(step) + +CRC_FUNC_PRE_LE(Z7_CRC_NUM_TABLES_USE) { const Byte *p = (const Byte *)data; - for (; size > 0 && ((unsigned)(ptrdiff_t)p & 7) != 0; size--, p++) + const Byte *lim; + for (; size && ((unsigned)(ptrdiff_t)p & (7 - (Z7_CRC_NUM_TABLES_USE & 4))) != 0; size--, p++) v = CRC_UPDATE_BYTE_2(v, *p); - for (; size >= 8; size -= 8, p += 8) + lim = p + size; + if (size >= Z7_CRC_NUM_TABLES_USE) { - UInt32 d; - v ^= *(const UInt32 *)(const void *)p; - v = - (table + 0x700)[((v ) & 0xFF)] - ^ (table + 0x600)[((v >> 8) & 0xFF)] - ^ (table + 0x500)[((v >> 16) & 0xFF)] - ^ (table + 0x400)[((v >> 24))]; - d = *((const UInt32 *)(const void *)p + 1); - v ^= - (table + 0x300)[((d ) & 0xFF)] - ^ (table + 0x200)[((d >> 8) & 0xFF)] - ^ (table + 0x100)[((d >> 16) & 0xFF)] - ^ (table + 0x000)[((d >> 24))]; + lim -= Z7_CRC_NUM_TABLES_USE; + do + { + v ^= R(0); + { +#if Z7_CRC_NUM_TABLES_USE == 1 * 4 + v = Q(0, v); +#else +#define U2(r, op) \ + { d = R(r); x op Q(Z7_CRC_NUM_TABLES_USE / 4 - 1 - (r), d); } + UInt32 d, x; + U2(1, =) +#if Z7_CRC_NUM_TABLES_USE >= 3 * 4 +#define U(r) U2(r, ^=) + U(2) +#if Z7_CRC_NUM_TABLES_USE >= 4 * 4 + U(3) +#if Z7_CRC_NUM_TABLES_USE >= 5 * 4 + U(4) +#if Z7_CRC_NUM_TABLES_USE >= 6 * 4 + U(5) +#if Z7_CRC_NUM_TABLES_USE >= 7 * 4 +#error Stop_Compiling_Bad_Z7_CRC_NUM_TABLES +#endif +#endif +#endif +#endif +#endif +#undef U +#undef U2 + v = x ^ Q(Z7_CRC_NUM_TABLES_USE / 4 - 1, v); +#endif + } + p += Z7_CRC_NUM_TABLES_USE; + } + while (p <= lim); + lim += Z7_CRC_NUM_TABLES_USE; } - for (; size > 0; size--, p++) + for (; p < lim; p++) v = CRC_UPDATE_BYTE_2(v, *p); return v; } +#undef CRC_UPDATE_BYTE_2 +#undef R +#undef Q +#undef CRC_FUNC_PRE_LE +#undef CRC_FUNC_PRE_LE2 + #endif + + #ifndef MY_CPU_LE -#define CRC_UINT32_SWAP(v) Z7_BSWAP32(v) +#define CRC_UPDATE_BYTE_2_BE(crc, b) (table[((crc) >> 24) ^ (b)] ^ ((crc) << 8)) -#define CRC_UPDATE_BYTE_2_BE(crc, b) (table[(((crc) >> 24) ^ (b))] ^ ((crc) << 8)) +#define Q(n, d) \ + ( (table + ((n) * 4 + 0) * 0x100)[((d)) & 0xFF] \ + ^ (table + ((n) * 4 + 1) * 0x100)[((d) >> 1 * 8) & 0xFF] \ + ^ (table + ((n) * 4 + 2) * 0x100)[((d) >> 2 * 8) & 0xFF] \ + ^ (table + ((n) * 4 + 3) * 0x100)[((d) >> 3 * 8)] ) -UInt32 Z7_FASTCALL CrcUpdateT1_BeT4(UInt32 v, const void *data, size_t size, const UInt32 *table) -{ - const Byte *p = (const Byte *)data; - table += 0x100; - v = CRC_UINT32_SWAP(v); - for (; size > 0 && ((unsigned)(ptrdiff_t)p & 3) != 0; size--, p++) - v = CRC_UPDATE_BYTE_2_BE(v, *p); - for (; size >= 4; size -= 4, p += 4) - { - v ^= *(const UInt32 *)(const void *)p; - v = - (table + 0x000)[((v ) & 0xFF)] - ^ (table + 0x100)[((v >> 8) & 0xFF)] - ^ (table + 0x200)[((v >> 16) & 0xFF)] - ^ (table + 0x300)[((v >> 24))]; - } - for (; size > 0; size--, p++) - v = CRC_UPDATE_BYTE_2_BE(v, *p); - return CRC_UINT32_SWAP(v); -} +#ifdef Z7_CRC_DEBUG_BE + #define R(a) GetBe32a((const UInt32 *)(const void *)p + (a)) +#else + #define R(a) *((const UInt32 *)(const void *)p + (a)) +#endif + + +#define CRC_FUNC_PRE_BE2(step) \ +UInt32 Z7_FASTCALL CrcUpdateT1_BeT ## step (UInt32 v, const void *data, size_t size, const UInt32 *table) -UInt32 Z7_FASTCALL CrcUpdateT1_BeT8(UInt32 v, const void *data, size_t size, const UInt32 *table) +#define CRC_FUNC_PRE_BE(step) \ + CRC_FUNC_PRE_BE2(step); \ + CRC_FUNC_PRE_BE2(step) + +CRC_FUNC_PRE_BE(Z7_CRC_NUM_TABLES_USE) { const Byte *p = (const Byte *)data; + const Byte *lim; table += 0x100; - v = CRC_UINT32_SWAP(v); - for (; size > 0 && ((unsigned)(ptrdiff_t)p & 7) != 0; size--, p++) + v = Z7_BSWAP32(v); + for (; size && ((unsigned)(ptrdiff_t)p & (7 - (Z7_CRC_NUM_TABLES_USE & 4))) != 0; size--, p++) v = CRC_UPDATE_BYTE_2_BE(v, *p); - for (; size >= 8; size -= 8, p += 8) + lim = p + size; + if (size >= Z7_CRC_NUM_TABLES_USE) { - UInt32 d; - v ^= *(const UInt32 *)(const void *)p; - v = - (table + 0x400)[((v ) & 0xFF)] - ^ (table + 0x500)[((v >> 8) & 0xFF)] - ^ (table + 0x600)[((v >> 16) & 0xFF)] - ^ (table + 0x700)[((v >> 24))]; - d = *((const UInt32 *)(const void *)p + 1); - v ^= - (table + 0x000)[((d ) & 0xFF)] - ^ (table + 0x100)[((d >> 8) & 0xFF)] - ^ (table + 0x200)[((d >> 16) & 0xFF)] - ^ (table + 0x300)[((d >> 24))]; + lim -= Z7_CRC_NUM_TABLES_USE; + do + { + v ^= R(0); + { +#if Z7_CRC_NUM_TABLES_USE == 1 * 4 + v = Q(0, v); +#else +#define U2(r, op) \ + { d = R(r); x op Q(Z7_CRC_NUM_TABLES_USE / 4 - 1 - (r), d); } + UInt32 d, x; + U2(1, =) +#if Z7_CRC_NUM_TABLES_USE >= 3 * 4 +#define U(r) U2(r, ^=) + U(2) +#if Z7_CRC_NUM_TABLES_USE >= 4 * 4 + U(3) +#if Z7_CRC_NUM_TABLES_USE >= 5 * 4 + U(4) +#if Z7_CRC_NUM_TABLES_USE >= 6 * 4 + U(5) +#if Z7_CRC_NUM_TABLES_USE >= 7 * 4 +#error Stop_Compiling_Bad_Z7_CRC_NUM_TABLES +#endif +#endif +#endif +#endif +#endif +#undef U +#undef U2 + v = x ^ Q(Z7_CRC_NUM_TABLES_USE / 4 - 1, v); +#endif + } + p += Z7_CRC_NUM_TABLES_USE; + } + while (p <= lim); + lim += Z7_CRC_NUM_TABLES_USE; } - for (; size > 0; size--, p++) + for (; p < lim; p++) v = CRC_UPDATE_BYTE_2_BE(v, *p); - return CRC_UINT32_SWAP(v); + return Z7_BSWAP32(v); } +#undef CRC_UPDATE_BYTE_2_BE +#undef R +#undef Q +#undef CRC_FUNC_PRE_BE +#undef CRC_FUNC_PRE_BE2 + +#endif +#undef Z7_CRC_NUM_TABLES_USE #endif diff --git a/C/7zDec.c b/C/7zDec.c index 96c6035..c9b4064 100644 --- a/C/7zDec.c +++ b/C/7zDec.c @@ -1,5 +1,5 @@ /* 7zDec.c -- Decoding from 7z folder -2023-04-02 : Igor Pavlov : Public domain */ +2024-03-01 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -51,6 +51,7 @@ #ifndef Z7_NO_METHODS_FILTERS #define k_Delta 3 +#define k_RISCV 0xb #define k_BCJ 0x3030103 #define k_PPC 0x3030205 #define k_IA64 0x3030401 @@ -362,6 +363,7 @@ static SRes CheckSupportedFolder(const CSzFolder *f) case k_IA64: case k_SPARC: case k_ARM: + case k_RISCV: #endif #ifdef Z7_USE_FILTER_ARM64 case k_ARM64: @@ -535,10 +537,10 @@ static SRes SzFolder_Decode2(const CSzFolder *folder, } } } - #if defined(Z7_USE_BRANCH_FILTER) +#if defined(Z7_USE_BRANCH_FILTER) else if (ci == 1) { - #if !defined(Z7_NO_METHODS_FILTERS) +#if !defined(Z7_NO_METHODS_FILTERS) if (coder->MethodID == k_Delta) { if (coder->PropsSize != 1) @@ -550,22 +552,43 @@ static SRes SzFolder_Decode2(const CSzFolder *folder, } continue; } - #endif +#endif - #ifdef Z7_USE_FILTER_ARM64 +#ifdef Z7_USE_FILTER_ARM64 if (coder->MethodID == k_ARM64) { UInt32 pc = 0; if (coder->PropsSize == 4) + { pc = GetUi32(propsData + coder->PropsOffset); + if (pc & 3) + return SZ_ERROR_UNSUPPORTED; + } else if (coder->PropsSize != 0) return SZ_ERROR_UNSUPPORTED; z7_BranchConv_ARM64_Dec(outBuffer, outSize, pc); continue; } - #endif - - #if !defined(Z7_NO_METHODS_FILTERS) || defined(Z7_USE_FILTER_ARMT) +#endif + +#if !defined(Z7_NO_METHODS_FILTERS) + if (coder->MethodID == k_RISCV) + { + UInt32 pc = 0; + if (coder->PropsSize == 4) + { + pc = GetUi32(propsData + coder->PropsOffset); + if (pc & 1) + return SZ_ERROR_UNSUPPORTED; + } + else if (coder->PropsSize != 0) + return SZ_ERROR_UNSUPPORTED; + z7_BranchConv_RISCV_Dec(outBuffer, outSize, pc); + continue; + } +#endif + +#if !defined(Z7_NO_METHODS_FILTERS) || defined(Z7_USE_FILTER_ARMT) { if (coder->PropsSize != 0) return SZ_ERROR_UNSUPPORTED; @@ -579,7 +602,8 @@ static SRes SzFolder_Decode2(const CSzFolder *folder, z7_BranchConvSt_X86_Dec(outBuffer, outSize, 0, &state); // pc = 0 break; } - CASE_BRA_CONV(PPC) + case k_PPC: Z7_BRANCH_CONV_DEC_2(BranchConv_PPC)(outBuffer, outSize, 0); break; // pc = 0; + // CASE_BRA_CONV(PPC) CASE_BRA_CONV(IA64) CASE_BRA_CONV(SPARC) CASE_BRA_CONV(ARM) @@ -592,9 +616,9 @@ static SRes SzFolder_Decode2(const CSzFolder *folder, } continue; } - #endif +#endif } // (c == 1) - #endif +#endif // Z7_USE_BRANCH_FILTER else return SZ_ERROR_UNSUPPORTED; } diff --git a/C/7zTypes.h b/C/7zTypes.h index 1fcb247..5b77420 100644 --- a/C/7zTypes.h +++ b/C/7zTypes.h @@ -1,5 +1,5 @@ /* 7zTypes.h -- Basic types -2023-04-02 : Igor Pavlov : Public domain */ +2024-01-24 : Igor Pavlov : Public domain */ #ifndef ZIP7_7Z_TYPES_H #define ZIP7_7Z_TYPES_H @@ -530,20 +530,20 @@ struct ISzAlloc #define Z7_CONTAINER_FROM_VTBL_CLS(ptr, type, m) Z7_CONTAINER_FROM_VTBL(ptr, type, m) */ #if defined (__clang__) || defined(__GNUC__) -#define Z7_DIAGNOSCTIC_IGNORE_BEGIN_CAST_QUAL \ +#define Z7_DIAGNOSTIC_IGNORE_BEGIN_CAST_QUAL \ _Pragma("GCC diagnostic push") \ _Pragma("GCC diagnostic ignored \"-Wcast-qual\"") -#define Z7_DIAGNOSCTIC_IGNORE_END_CAST_QUAL \ +#define Z7_DIAGNOSTIC_IGNORE_END_CAST_QUAL \ _Pragma("GCC diagnostic pop") #else -#define Z7_DIAGNOSCTIC_IGNORE_BEGIN_CAST_QUAL -#define Z7_DIAGNOSCTIC_IGNORE_END_CAST_QUAL +#define Z7_DIAGNOSTIC_IGNORE_BEGIN_CAST_QUAL +#define Z7_DIAGNOSTIC_IGNORE_END_CAST_QUAL #endif #define Z7_CONTAINER_FROM_VTBL_TO_DECL_VAR(ptr, type, m, p) \ - Z7_DIAGNOSCTIC_IGNORE_BEGIN_CAST_QUAL \ + Z7_DIAGNOSTIC_IGNORE_BEGIN_CAST_QUAL \ type *p = Z7_CONTAINER_FROM_VTBL(ptr, type, m); \ - Z7_DIAGNOSCTIC_IGNORE_END_CAST_QUAL + Z7_DIAGNOSTIC_IGNORE_END_CAST_QUAL #define Z7_CONTAINER_FROM_VTBL_TO_DECL_VAR_pp_vt_p(type) \ Z7_CONTAINER_FROM_VTBL_TO_DECL_VAR(pp, type, vt, p) diff --git a/C/7zVersion.h b/C/7zVersion.h index 7549239..72b915a 100644 --- a/C/7zVersion.h +++ b/C/7zVersion.h @@ -1,7 +1,7 @@ -#define MY_VER_MAJOR 23 -#define MY_VER_MINOR 01 +#define MY_VER_MAJOR 24 +#define MY_VER_MINOR 05 #define MY_VER_BUILD 0 -#define MY_VERSION_NUMBERS "23.01" +#define MY_VERSION_NUMBERS "24.05" #define MY_VERSION MY_VERSION_NUMBERS #ifdef MY_CPU_NAME @@ -10,12 +10,12 @@ #define MY_VERSION_CPU MY_VERSION #endif -#define MY_DATE "2023-06-20" +#define MY_DATE "2024-05-14" #undef MY_COPYRIGHT #undef MY_VERSION_COPYRIGHT_DATE #define MY_AUTHOR_NAME "Igor Pavlov" #define MY_COPYRIGHT_PD "Igor Pavlov : Public domain" -#define MY_COPYRIGHT_CR "Copyright (c) 1999-2023 Igor Pavlov" +#define MY_COPYRIGHT_CR "Copyright (c) 1999-2024 Igor Pavlov" #ifdef USE_COPYRIGHT_CR #define MY_COPYRIGHT MY_COPYRIGHT_CR diff --git a/C/7zip_gcc_c.mak b/C/7zip_gcc_c.mak index f19a99b..195d23d 100644 --- a/C/7zip_gcc_c.mak +++ b/C/7zip_gcc_c.mak @@ -22,8 +22,8 @@ CFLAGS_BASE_LIST = -c # for ASM file # CFLAGS_BASE_LIST = -S -FLAGS_FLTO = FLAGS_FLTO = -flto +FLAGS_FLTO = CFLAGS_BASE = $(MY_ARCH_2) -O2 $(CFLAGS_BASE_LIST) $(CFLAGS_WARN_WALL) $(CFLAGS_WARN) \ -DNDEBUG -D_REENTRANT -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE @@ -329,7 +329,7 @@ endif ifdef IS_ARM64 $O/LzmaDecOpt.o: ../../../Asm/arm64/LzmaDecOpt.S ../../../Asm/arm64/7zAsm.S - $(CC) $(CFLAGS) $< + $(CC) $(CFLAGS) $(ASM_FLAGS) $< endif $O/LzmaDec.o: ../../LzmaDec.c diff --git a/C/Aes.c b/C/Aes.c index bcaafab..abc5d24 100644 --- a/C/Aes.c +++ b/C/Aes.c @@ -1,5 +1,5 @@ /* Aes.c -- AES encryption / decryption -2023-04-02 : Igor Pavlov : Public domain */ +2024-03-01 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -13,7 +13,9 @@ AES_CODE_FUNC g_AesCtr_Code; UInt32 g_Aes_SupportedFunctions_Flags; #endif +MY_ALIGN(64) static UInt32 T[256 * 4]; +MY_ALIGN(64) static const Byte Sbox[256] = { 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76, 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0, 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0, @@ -33,7 +35,9 @@ static const Byte Sbox[256] = { 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16}; +MY_ALIGN(64) static UInt32 D[256 * 4]; +MY_ALIGN(64) static Byte InvS[256]; #define xtime(x) ((((x) << 1) ^ (((x) & 0x80) != 0 ? 0x1B : 0)) & 0xFF) @@ -54,24 +58,54 @@ static Byte InvS[256]; // #define Z7_SHOW_AES_STATUS #ifdef MY_CPU_X86_OR_AMD64 - #define USE_HW_AES -#elif defined(MY_CPU_ARM_OR_ARM64) && defined(MY_CPU_LE) - #if defined(__clang__) - #if (__clang_major__ >= 8) // fix that check - #define USE_HW_AES - #endif - #elif defined(__GNUC__) - #if (__GNUC__ >= 6) // fix that check + + #if defined(__INTEL_COMPILER) + #if (__INTEL_COMPILER >= 1110) #define USE_HW_AES + #if (__INTEL_COMPILER >= 1900) + #define USE_HW_VAES + #endif #endif + #elif defined(Z7_CLANG_VERSION) && (Z7_CLANG_VERSION >= 30800) \ + || defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 40400) + #define USE_HW_AES + #if defined(__clang__) && (__clang_major__ >= 8) \ + || defined(__GNUC__) && (__GNUC__ >= 8) + #define USE_HW_VAES + #endif #elif defined(_MSC_VER) - #if _MSC_VER >= 1910 + #define USE_HW_AES + #define USE_HW_VAES + #endif + +#elif defined(MY_CPU_ARM_OR_ARM64) && defined(MY_CPU_LE) + + #if defined(__ARM_FEATURE_AES) \ + || defined(__ARM_FEATURE_CRYPTO) + #define USE_HW_AES + #else + #if defined(MY_CPU_ARM64) \ + || defined(__ARM_ARCH) && (__ARM_ARCH >= 4) \ + || defined(Z7_MSC_VER_ORIGINAL) + #if defined(__ARM_FP) && \ + ( defined(Z7_CLANG_VERSION) && (Z7_CLANG_VERSION >= 30800) \ + || defined(__GNUC__) && (__GNUC__ >= 6) \ + ) \ + || defined(Z7_MSC_VER_ORIGINAL) && (_MSC_VER >= 1910) + #if defined(MY_CPU_ARM64) \ + || !defined(Z7_CLANG_VERSION) \ + || defined(__ARM_NEON) && \ + (Z7_CLANG_VERSION < 170000 || \ + Z7_CLANG_VERSION > 170001) #define USE_HW_AES #endif + #endif + #endif #endif #endif #ifdef USE_HW_AES +// #pragma message("=== Aes.c USE_HW_AES === ") #ifdef Z7_SHOW_AES_STATUS #include #define PRF(x) x @@ -136,6 +170,7 @@ void AesGenTables(void) #endif #ifdef MY_CPU_X86_OR_AMD64 + #ifdef USE_HW_VAES if (CPU_IsSupported_VAES_AVX2()) { PRF(printf("\n===vaes avx2\n")); @@ -146,6 +181,7 @@ void AesGenTables(void) #endif } #endif + #endif } #endif diff --git a/C/AesOpt.c b/C/AesOpt.c index cfa6413..58769ea 100644 --- a/C/AesOpt.c +++ b/C/AesOpt.c @@ -1,5 +1,5 @@ /* AesOpt.c -- AES optimized code for x86 AES hardware instructions -2023-04-02 : Igor Pavlov : Public domain */ +2024-03-01 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -15,8 +15,8 @@ #define USE_INTEL_VAES #endif #endif - #elif defined(__clang__) && (__clang_major__ > 3 || __clang_major__ == 3 && __clang_minor__ >= 8) \ - || defined(__GNUC__) && (__GNUC__ > 4 || __GNUC__ == 4 && __GNUC_MINOR__ >= 4) + #elif defined(Z7_CLANG_VERSION) && (Z7_CLANG_VERSION >= 30800) \ + || defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 40400) #define USE_INTEL_AES #if !defined(__AES__) #define ATTRIB_AES __attribute__((__target__("aes"))) @@ -35,27 +35,37 @@ #define USE_INTEL_VAES #endif #endif + #ifndef USE_INTEL_AES + #define Z7_USE_AES_HW_STUB + #endif + #ifndef USE_INTEL_VAES + #define Z7_USE_VAES_HW_STUB + #endif #endif -#ifndef ATTRIB_AES - #define ATTRIB_AES -#endif -#ifndef ATTRIB_VAES - #define ATTRIB_VAES -#endif + #ifndef USE_INTEL_AES + // #define Z7_USE_AES_HW_STUB // for debug + #endif + #ifndef USE_INTEL_VAES + // #define Z7_USE_VAES_HW_STUB // for debug + #endif #ifdef USE_INTEL_AES #include -#ifndef USE_INTEL_VAES +#if !defined(USE_INTEL_VAES) && defined(Z7_USE_VAES_HW_STUB) #define AES_TYPE_keys UInt32 #define AES_TYPE_data Byte // #define AES_TYPE_keys __m128i // #define AES_TYPE_data __m128i #endif +#ifndef ATTRIB_AES + #define ATTRIB_AES +#endif + #define AES_FUNC_START(name) \ void Z7_FASTCALL name(UInt32 *ivAes, Byte *data8, size_t numBlocks) // void Z7_FASTCALL name(__m128i *p, __m128i *data, size_t numBlocks) @@ -69,8 +79,6 @@ AES_FUNC_START (name) #define MM_OP_m(op, src) MM_OP(op, m, src) #define MM_XOR( dest, src) MM_OP(_mm_xor_si128, dest, src) -#define AVX_XOR(dest, src) MM_OP(_mm256_xor_si256, dest, src) - AES_FUNC_START2 (AesCbc_Encode_HW) { @@ -139,11 +147,6 @@ AES_FUNC_START2 (AesCbc_Encode_HW) #define XOR_data_M1(reg, ii) MM_XOR (reg, data[ii- 1]) #endif -#define AVX_DECLARE_VAR(reg, ii) __m256i reg; -#define AVX_LOAD_data( reg, ii) reg = ((const __m256i *)(const void *)data)[ii]; -#define AVX_STORE_data( reg, ii) ((__m256i *)(void *)data)[ii] = reg; -#define AVX_XOR_data_M1(reg, ii) AVX_XOR (reg, (((const __m256i *)(const void *)(data - 1))[ii])) - #define MM_OP_key(op, reg) MM_OP(op, reg, key); #define AES_DEC( reg, ii) MM_OP_key (_mm_aesdec_si128, reg) @@ -152,27 +155,13 @@ AES_FUNC_START2 (AesCbc_Encode_HW) #define AES_ENC_LAST( reg, ii) MM_OP_key (_mm_aesenclast_si128, reg) #define AES_XOR( reg, ii) MM_OP_key (_mm_xor_si128, reg) - -#define AVX_AES_DEC( reg, ii) MM_OP_key (_mm256_aesdec_epi128, reg) -#define AVX_AES_DEC_LAST( reg, ii) MM_OP_key (_mm256_aesdeclast_epi128, reg) -#define AVX_AES_ENC( reg, ii) MM_OP_key (_mm256_aesenc_epi128, reg) -#define AVX_AES_ENC_LAST( reg, ii) MM_OP_key (_mm256_aesenclast_epi128, reg) -#define AVX_AES_XOR( reg, ii) MM_OP_key (_mm256_xor_si256, reg) - #define CTR_START(reg, ii) MM_OP (_mm_add_epi64, ctr, one) reg = ctr; #define CTR_END( reg, ii) MM_XOR (data[ii], reg) -#define AVX_CTR_START(reg, ii) MM_OP (_mm256_add_epi64, ctr2, two) reg = _mm256_xor_si256(ctr2, key); -#define AVX_CTR_END( reg, ii) AVX_XOR (((__m256i *)(void *)data)[ii], reg) - #define WOP_KEY(op, n) { \ const __m128i key = w[n]; \ WOP(op); } -#define AVX_WOP_KEY(op, n) { \ - const __m256i key = w[n]; \ - WOP(op); } - #define WIDE_LOOP_START \ dataEnd = data + numBlocks; \ @@ -190,6 +179,40 @@ AES_FUNC_START2 (AesCbc_Encode_HW) for (; data < dataEnd; data++) + +#ifdef USE_INTEL_VAES + +#define AVX_XOR(dest, src) MM_OP(_mm256_xor_si256, dest, src) +#define AVX_DECLARE_VAR(reg, ii) __m256i reg; +#define AVX_LOAD_data( reg, ii) reg = ((const __m256i *)(const void *)data)[ii]; +#define AVX_STORE_data( reg, ii) ((__m256i *)(void *)data)[ii] = reg; +/* +AVX_XOR_data_M1() needs unaligned memory load +if (we don't use _mm256_loadu_si256() here) +{ + Most compilers with enabled optimizations generate fused AVX (LOAD + OP) + instruction that can load unaligned data. + But GCC and CLANG without -O2 or -O1 optimizations can generate separated + LOAD-ALIGNED (vmovdqa) instruction that will fail on execution. +} +Note: some compilers generate more instructions, if we use _mm256_loadu_si256() here. +v23.02: we use _mm256_loadu_si256() here, because we need compatibility with any compiler. +*/ +#define AVX_XOR_data_M1(reg, ii) AVX_XOR (reg, _mm256_loadu_si256(&(((const __m256i *)(const void *)(data - 1))[ii]))) +// for debug only: the following code will fail on execution, if compiled by some compilers: +// #define AVX_XOR_data_M1(reg, ii) AVX_XOR (reg, (((const __m256i *)(const void *)(data - 1))[ii])) + +#define AVX_AES_DEC( reg, ii) MM_OP_key (_mm256_aesdec_epi128, reg) +#define AVX_AES_DEC_LAST( reg, ii) MM_OP_key (_mm256_aesdeclast_epi128, reg) +#define AVX_AES_ENC( reg, ii) MM_OP_key (_mm256_aesenc_epi128, reg) +#define AVX_AES_ENC_LAST( reg, ii) MM_OP_key (_mm256_aesenclast_epi128, reg) +#define AVX_AES_XOR( reg, ii) MM_OP_key (_mm256_xor_si256, reg) +#define AVX_CTR_START(reg, ii) MM_OP (_mm256_add_epi64, ctr2, two) reg = _mm256_xor_si256(ctr2, key); +#define AVX_CTR_END( reg, ii) AVX_XOR (((__m256i *)(void *)data)[ii], reg) +#define AVX_WOP_KEY(op, n) { \ + const __m256i key = w[n]; \ + WOP(op); } + #define NUM_AES_KEYS_MAX 15 #define WIDE_LOOP_START_AVX(OP) \ @@ -214,6 +237,9 @@ AES_FUNC_START2 (AesCbc_Encode_HW) /* MSVC for x86: If we don't call _mm256_zeroupper(), and -arch:IA32 is not specified, MSVC still can insert vzeroupper instruction. */ +#endif + + AES_FUNC_START2 (AesCbc_Decode_HW) { @@ -380,6 +406,9 @@ required that must be included before . #endif #endif // __clang__ && _MSC_VER +#ifndef ATTRIB_VAES + #define ATTRIB_VAES +#endif #define VAES_FUNC_START2(name) \ AES_FUNC_START (name); \ @@ -519,10 +548,18 @@ VAES_FUNC_START2 (AesCtr_Code_HW_256) /* no USE_INTEL_AES */ +#if defined(Z7_USE_AES_HW_STUB) +// We can compile this file with another C compiler, +// or we can compile asm version. +// So we can generate real code instead of this stub function. +// #if defined(_MSC_VER) #pragma message("AES HW_SW stub was used") +// #endif +#if !defined(USE_INTEL_VAES) && defined(Z7_USE_VAES_HW_STUB) #define AES_TYPE_keys UInt32 #define AES_TYPE_data Byte +#endif #define AES_FUNC_START(name) \ void Z7_FASTCALL name(UInt32 *p, Byte *data, size_t numBlocks) \ @@ -535,13 +572,16 @@ VAES_FUNC_START2 (AesCtr_Code_HW_256) AES_COMPAT_STUB (AesCbc_Encode) AES_COMPAT_STUB (AesCbc_Decode) AES_COMPAT_STUB (AesCtr_Code) +#endif // Z7_USE_AES_HW_STUB #endif // USE_INTEL_AES #ifndef USE_INTEL_VAES - +#if defined(Z7_USE_VAES_HW_STUB) +// #if defined(_MSC_VER) #pragma message("VAES HW_SW stub was used") +// #endif #define VAES_COMPAT_STUB(name) \ void Z7_FASTCALL name ## _256(UInt32 *p, Byte *data, size_t numBlocks); \ @@ -550,36 +590,59 @@ AES_COMPAT_STUB (AesCtr_Code) VAES_COMPAT_STUB (AesCbc_Decode_HW) VAES_COMPAT_STUB (AesCtr_Code_HW) - +#endif #endif // ! USE_INTEL_VAES + + #elif defined(MY_CPU_ARM_OR_ARM64) && defined(MY_CPU_LE) - #if defined(__clang__) - #if (__clang_major__ >= 8) // fix that check + #if defined(__ARM_FEATURE_AES) \ + || defined(__ARM_FEATURE_CRYPTO) + #define USE_HW_AES + #else + #if defined(MY_CPU_ARM64) \ + || defined(__ARM_ARCH) && (__ARM_ARCH >= 4) \ + || defined(Z7_MSC_VER_ORIGINAL) + #if defined(__ARM_FP) && \ + ( defined(Z7_CLANG_VERSION) && (Z7_CLANG_VERSION >= 30800) \ + || defined(__GNUC__) && (__GNUC__ >= 6) \ + ) \ + || defined(Z7_MSC_VER_ORIGINAL) && (_MSC_VER >= 1910) + #if defined(MY_CPU_ARM64) \ + || !defined(Z7_CLANG_VERSION) \ + || defined(__ARM_NEON) && \ + (Z7_CLANG_VERSION < 170000 || \ + Z7_CLANG_VERSION > 170001) #define USE_HW_AES #endif - #elif defined(__GNUC__) - #if (__GNUC__ >= 6) // fix that check - #define USE_HW_AES #endif - #elif defined(_MSC_VER) - #if _MSC_VER >= 1910 - #define USE_HW_AES #endif #endif #ifdef USE_HW_AES // #pragma message("=== AES HW === ") +// __ARM_FEATURE_CRYPTO macro is deprecated in favor of the finer grained feature macro __ARM_FEATURE_AES #if defined(__clang__) || defined(__GNUC__) +#if !defined(__ARM_FEATURE_AES) && \ + !defined(__ARM_FEATURE_CRYPTO) #ifdef MY_CPU_ARM64 +#if defined(__clang__) + #define ATTRIB_AES __attribute__((__target__("crypto"))) +#else #define ATTRIB_AES __attribute__((__target__("+crypto"))) +#endif #else +#if defined(__clang__) + #define ATTRIB_AES __attribute__((__target__("armv8-a,aes"))) +#else #define ATTRIB_AES __attribute__((__target__("fpu=crypto-neon-fp-armv8"))) +#endif #endif +#endif #else // _MSC_VER // for arm32 @@ -590,12 +653,60 @@ VAES_COMPAT_STUB (AesCtr_Code_HW) #define ATTRIB_AES #endif -#if defined(_MSC_VER) && defined(MY_CPU_ARM64) +#if defined(Z7_MSC_VER_ORIGINAL) && defined(MY_CPU_ARM64) #include #else +/* + clang-17.0.1: error : Cannot select: intrinsic %llvm.arm.neon.aese + clang + 3.8.1 : __ARM_NEON : defined(__ARM_FEATURE_CRYPTO) + 7.0.1 : __ARM_NEON : __ARM_ARCH >= 8 && defined(__ARM_FEATURE_CRYPTO) + 11.?.0 : __ARM_NEON && __ARM_FP : __ARM_ARCH >= 8 && defined(__ARM_FEATURE_CRYPTO) + 13.0.1 : __ARM_NEON && __ARM_FP : __ARM_ARCH >= 8 && defined(__ARM_FEATURE_AES) + 16 : __ARM_NEON && __ARM_FP : __ARM_ARCH >= 8 +*/ +#if defined(__clang__) && __clang_major__ < 16 +#if !defined(__ARM_FEATURE_AES) && \ + !defined(__ARM_FEATURE_CRYPTO) +// #pragma message("=== we set __ARM_FEATURE_CRYPTO 1 === ") + Z7_DIAGNOSTIC_IGNORE_BEGIN_RESERVED_MACRO_IDENTIFIER + #define Z7_ARM_FEATURE_CRYPTO_WAS_SET 1 +// #if defined(__clang__) && __clang_major__ < 13 + #define __ARM_FEATURE_CRYPTO 1 +// #else + #define __ARM_FEATURE_AES 1 +// #endif + Z7_DIAGNOSTIC_IGNORE_END_RESERVED_MACRO_IDENTIFIER +#endif +#endif // clang + +#if defined(__clang__) + +#if defined(__ARM_ARCH) && __ARM_ARCH < 8 + Z7_DIAGNOSTIC_IGNORE_BEGIN_RESERVED_MACRO_IDENTIFIER +// #pragma message("#define __ARM_ARCH 8") + #undef __ARM_ARCH + #define __ARM_ARCH 8 + Z7_DIAGNOSTIC_IGNORE_END_RESERVED_MACRO_IDENTIFIER +#endif + +#endif // clang + #include + +#if defined(Z7_ARM_FEATURE_CRYPTO_WAS_SET) && \ + defined(__ARM_FEATURE_CRYPTO) && \ + defined(__ARM_FEATURE_AES) +Z7_DIAGNOSTIC_IGNORE_BEGIN_RESERVED_MACRO_IDENTIFIER + #undef __ARM_FEATURE_CRYPTO + #undef __ARM_FEATURE_AES + #undef Z7_ARM_FEATURE_CRYPTO_WAS_SET +Z7_DIAGNOSTIC_IGNORE_END_RESERVED_MACRO_IDENTIFIER +// #pragma message("=== we undefine __ARM_FEATURE_CRYPTO === ") #endif +#endif // Z7_MSC_VER_ORIGINAL + typedef uint8x16_t v128; #define AES_FUNC_START(name) \ @@ -620,7 +731,7 @@ AES_FUNC_START (name) AES_FUNC_START2 (AesCbc_Encode_HW) { - v128 *p = (v128*)(void*)ivAes; + v128 * const p = (v128*)(void*)ivAes; v128 *data = (v128*)(void*)data8; v128 m = *p; const v128 k0 = p[2]; @@ -639,7 +750,7 @@ AES_FUNC_START2 (AesCbc_Encode_HW) const v128 k_z0 = w[2]; for (; numBlocks != 0; numBlocks--, data++) { - MM_XOR_m (*data); + MM_XOR_m (*data) AES_E_MC_m (k0) AES_E_MC_m (k1) AES_E_MC_m (k2) @@ -660,7 +771,7 @@ AES_FUNC_START2 (AesCbc_Encode_HW) } } AES_E_m (k_z1) - MM_XOR_m (k_z0); + MM_XOR_m (k_z0) *data = m; } *p = m; @@ -745,7 +856,7 @@ AES_FUNC_START2 (AesCbc_Decode_HW) while (w != p); WOP_KEY (AES_D, 1) WOP_KEY (AES_XOR, 0) - MM_XOR (m0, iv); + MM_XOR (m0, iv) WOP_M1 (XOR_data_M1) iv = data[NUM_WAYS - 1]; WOP (STORE_data) @@ -759,14 +870,14 @@ AES_FUNC_START2 (AesCbc_Decode_HW) AES_D_IMC_m (w[2]) do { - AES_D_IMC_m (w[1]); - AES_D_IMC_m (w[0]); + AES_D_IMC_m (w[1]) + AES_D_IMC_m (w[0]) w -= 2; } while (w != p); - AES_D_m (w[1]); - MM_XOR_m (w[0]); - MM_XOR_m (iv); + AES_D_m (w[1]) + MM_XOR_m (w[0]) + MM_XOR_m (iv) iv = *data; *data = m; } @@ -783,6 +894,12 @@ AES_FUNC_START2 (AesCtr_Code_HW) const v128 *wEnd = p + ((size_t)*(const UInt32 *)(p + 1)) * 2; const v128 *dataEnd; uint64x2_t one = vdupq_n_u64(0); + +// the bug in clang: +// __builtin_neon_vsetq_lane_i64(__s0, (int8x16_t)__s1, __p2); +#if defined(__clang__) && (__clang_major__ <= 9) +#pragma GCC diagnostic ignored "-Wvector-conversion" +#endif one = vsetq_lane_u64(1, one, 0); p += 2; @@ -809,11 +926,11 @@ AES_FUNC_START2 (AesCtr_Code_HW) { const v128 *w = p; v128 m; - CTR_START (m, 0); + CTR_START (m, 0) do { - AES_E_MC_m (w[0]); - AES_E_MC_m (w[1]); + AES_E_MC_m (w[0]) + AES_E_MC_m (w[1]) w += 2; } while (w != wEnd); diff --git a/C/Alloc.c b/C/Alloc.c index d841bf2..63e1a12 100644 --- a/C/Alloc.c +++ b/C/Alloc.c @@ -1,5 +1,5 @@ /* Alloc.c -- Memory allocation functions -2023-04-02 : Igor Pavlov : Public domain */ +2024-02-18 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -10,19 +10,18 @@ #include "Alloc.h" -#ifdef _WIN32 -#ifdef Z7_LARGE_PAGES -#if defined(__clang__) || defined(__GNUC__) -typedef void (*Z7_voidFunction)(void); -#define MY_CAST_FUNC (Z7_voidFunction) -#elif defined(_MSC_VER) && _MSC_VER > 1920 -#define MY_CAST_FUNC (void *) -// #pragma warning(disable : 4191) // 'type cast': unsafe conversion from 'FARPROC' to 'void (__cdecl *)()' -#else -#define MY_CAST_FUNC +#if defined(Z7_LARGE_PAGES) && defined(_WIN32) && \ + (!defined(Z7_WIN32_WINNT_MIN) || Z7_WIN32_WINNT_MIN < 0x0502) // < Win2003 (xp-64) + #define Z7_USE_DYN_GetLargePageMinimum +#endif + +// for debug: +#if 0 +#if defined(__CHERI__) && defined(__SIZEOF_POINTER__) && (__SIZEOF_POINTER__ == 16) +// #pragma message("=== Z7_ALLOC_NO_OFFSET_ALLOCATOR === ") +#define Z7_ALLOC_NO_OFFSET_ALLOCATOR +#endif #endif -#endif // Z7_LARGE_PAGES -#endif // _WIN32 // #define SZ_ALLOC_DEBUG /* #define SZ_ALLOC_DEBUG */ @@ -146,7 +145,9 @@ static void PrintAddr(void *p) #define PRINT_FREE(name, cnt, ptr) #define Print(s) #define PrintLn() +#ifndef Z7_ALLOC_NO_OFFSET_ALLOCATOR #define PrintHex(v, align) +#endif #define PrintAddr(p) #endif @@ -246,9 +247,9 @@ void MidFree(void *address) #ifdef Z7_LARGE_PAGES #ifdef MEM_LARGE_PAGES - #define MY__MEM_LARGE_PAGES MEM_LARGE_PAGES + #define MY_MEM_LARGE_PAGES MEM_LARGE_PAGES #else - #define MY__MEM_LARGE_PAGES 0x20000000 + #define MY_MEM_LARGE_PAGES 0x20000000 #endif extern @@ -258,19 +259,23 @@ typedef SIZE_T (WINAPI *Func_GetLargePageMinimum)(VOID); void SetLargePageSize(void) { - #ifdef Z7_LARGE_PAGES SIZE_T size; +#ifdef Z7_USE_DYN_GetLargePageMinimum +Z7_DIAGNOSTIC_IGNORE_CAST_FUNCTION + const Func_GetLargePageMinimum fn = - (Func_GetLargePageMinimum) MY_CAST_FUNC GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")), + (Func_GetLargePageMinimum) Z7_CAST_FUNC_C GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")), "GetLargePageMinimum"); if (!fn) return; size = fn(); +#else + size = GetLargePageMinimum(); +#endif if (size == 0 || (size & (size - 1)) != 0) return; g_LargePageSize = size; - #endif } #endif // Z7_LARGE_PAGES @@ -292,7 +297,7 @@ void *BigAlloc(size_t size) size2 = (size + ps) & ~ps; if (size2 >= size) { - void *p = VirtualAlloc(NULL, size2, MEM_COMMIT | MY__MEM_LARGE_PAGES, PAGE_READWRITE); + void *p = VirtualAlloc(NULL, size2, MEM_COMMIT | MY_MEM_LARGE_PAGES, PAGE_READWRITE); if (p) { PRINT_ALLOC("Alloc-BM ", g_allocCountMid, size2, p) @@ -328,20 +333,7 @@ const ISzAlloc g_MidAlloc = { SzMidAlloc, SzMidFree }; const ISzAlloc g_BigAlloc = { SzBigAlloc, SzBigFree }; #endif -/* - uintptr_t : C99 (optional) - : unsupported in VS6 -*/ - -#ifdef _WIN32 - typedef UINT_PTR UIntPtr; -#else - /* - typedef uintptr_t UIntPtr; - */ - typedef ptrdiff_t UIntPtr; -#endif - +#ifndef Z7_ALLOC_NO_OFFSET_ALLOCATOR #define ADJUST_ALLOC_SIZE 0 /* @@ -352,14 +344,36 @@ const ISzAlloc g_BigAlloc = { SzBigAlloc, SzBigFree }; MyAlloc() can return address that is NOT multiple of sizeof(void *). */ - /* -#define MY_ALIGN_PTR_DOWN(p, align) ((void *)((char *)(p) - ((size_t)(UIntPtr)(p) & ((align) - 1)))) + uintptr_t : C99 (optional) + : unsupported in VS6 */ -#define MY_ALIGN_PTR_DOWN(p, align) ((void *)((((UIntPtr)(p)) & ~((UIntPtr)(align) - 1)))) +typedef + #ifdef _WIN32 + UINT_PTR + #elif 1 + uintptr_t + #else + ptrdiff_t + #endif + MY_uintptr_t; + +#if 0 \ + || (defined(__CHERI__) \ + || defined(__SIZEOF_POINTER__) && (__SIZEOF_POINTER__ > 8)) +// for 128-bit pointers (cheri): +#define MY_ALIGN_PTR_DOWN(p, align) \ + ((void *)((char *)(p) - ((size_t)(MY_uintptr_t)(p) & ((align) - 1)))) +#else +#define MY_ALIGN_PTR_DOWN(p, align) \ + ((void *)((((MY_uintptr_t)(p)) & ~((MY_uintptr_t)(align) - 1)))) +#endif +#endif -#if !defined(_WIN32) && defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 200112L) +#if !defined(_WIN32) \ + && (defined(Z7_ALLOC_NO_OFFSET_ALLOCATOR) \ + || defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 200112L)) #define USE_posix_memalign #endif @@ -399,14 +413,13 @@ static int posix_memalign(void **ptr, size_t align, size_t size) #define ALLOC_ALIGN_SIZE ((size_t)1 << 7) -static void *SzAlignedAlloc(ISzAllocPtr pp, size_t size) +void *z7_AlignedAlloc(size_t size) { - #ifndef USE_posix_memalign +#ifndef USE_posix_memalign void *p; void *pAligned; size_t newSize; - UNUSED_VAR(pp) /* also we can allocate additional dummy ALLOC_ALIGN_SIZE bytes after aligned block to prevent cache line sharing with another allocated blocks */ @@ -431,10 +444,9 @@ static void *SzAlignedAlloc(ISzAllocPtr pp, size_t size) return pAligned; - #else +#else void *p; - UNUSED_VAR(pp) if (posix_memalign(&p, ALLOC_ALIGN_SIZE, size)) return NULL; @@ -443,19 +455,37 @@ static void *SzAlignedAlloc(ISzAllocPtr pp, size_t size) return p; - #endif +#endif +} + + +void z7_AlignedFree(void *address) +{ +#ifndef USE_posix_memalign + if (address) + MyFree(((void **)address)[-1]); +#else + free(address); +#endif +} + + +static void *SzAlignedAlloc(ISzAllocPtr pp, size_t size) +{ + UNUSED_VAR(pp) + return z7_AlignedAlloc(size); } static void SzAlignedFree(ISzAllocPtr pp, void *address) { UNUSED_VAR(pp) - #ifndef USE_posix_memalign +#ifndef USE_posix_memalign if (address) MyFree(((void **)address)[-1]); - #else +#else free(address); - #endif +#endif } @@ -463,16 +493,44 @@ const ISzAlloc g_AlignedAlloc = { SzAlignedAlloc, SzAlignedFree }; -#define MY_ALIGN_PTR_DOWN_1(p) MY_ALIGN_PTR_DOWN(p, sizeof(void *)) - /* we align ptr to support cases where CAlignOffsetAlloc::offset is not multiply of sizeof(void *) */ -#define REAL_BLOCK_PTR_VAR(p) ((void **)MY_ALIGN_PTR_DOWN_1(p))[-1] -/* -#define REAL_BLOCK_PTR_VAR(p) ((void **)(p))[-1] -*/ +#ifndef Z7_ALLOC_NO_OFFSET_ALLOCATOR +#if 1 + #define MY_ALIGN_PTR_DOWN_1(p) MY_ALIGN_PTR_DOWN(p, sizeof(void *)) + #define REAL_BLOCK_PTR_VAR(p) ((void **)MY_ALIGN_PTR_DOWN_1(p))[-1] +#else + // we can use this simplified code, + // if (CAlignOffsetAlloc::offset == (k * sizeof(void *)) + #define REAL_BLOCK_PTR_VAR(p) (((void **)(p))[-1]) +#endif +#endif + + +#if 0 +#ifndef Z7_ALLOC_NO_OFFSET_ALLOCATOR +#include +static void PrintPtr(const char *s, const void *p) +{ + const Byte *p2 = (const Byte *)&p; + unsigned i; + printf("%s %p ", s, p); + for (i = sizeof(p); i != 0;) + { + i--; + printf("%02x", p2[i]); + } + printf("\n"); +} +#endif +#endif + static void *AlignOffsetAlloc_Alloc(ISzAllocPtr pp, size_t size) { +#if defined(Z7_ALLOC_NO_OFFSET_ALLOCATOR) + UNUSED_VAR(pp) + return z7_AlignedAlloc(size); +#else const CAlignOffsetAlloc *p = Z7_CONTAINER_FROM_VTBL_CONST(pp, CAlignOffsetAlloc, vt); void *adr; void *pAligned; @@ -501,6 +559,12 @@ static void *AlignOffsetAlloc_Alloc(ISzAllocPtr pp, size_t size) pAligned = (char *)MY_ALIGN_PTR_DOWN((char *)adr + alignSize - p->offset + extra + ADJUST_ALLOC_SIZE, alignSize) + p->offset; +#if 0 + printf("\nalignSize = %6x, offset=%6x, size=%8x \n", (unsigned)alignSize, (unsigned)p->offset, (unsigned)size); + PrintPtr("base", adr); + PrintPtr("alig", pAligned); +#endif + PrintLn(); Print("- Aligned: "); Print(" size="); PrintHex(size, 8); @@ -512,11 +576,16 @@ static void *AlignOffsetAlloc_Alloc(ISzAllocPtr pp, size_t size) REAL_BLOCK_PTR_VAR(pAligned) = adr; return pAligned; +#endif } static void AlignOffsetAlloc_Free(ISzAllocPtr pp, void *address) { +#if defined(Z7_ALLOC_NO_OFFSET_ALLOCATOR) + UNUSED_VAR(pp) + z7_AlignedFree(address); +#else if (address) { const CAlignOffsetAlloc *p = Z7_CONTAINER_FROM_VTBL_CONST(pp, CAlignOffsetAlloc, vt); @@ -525,6 +594,7 @@ static void AlignOffsetAlloc_Free(ISzAllocPtr pp, void *address) PrintLn(); ISzAlloc_Free(p->baseAlloc, REAL_BLOCK_PTR_VAR(address)); } +#endif } diff --git a/C/Alloc.h b/C/Alloc.h index fac5b62..01bf6b7 100644 --- a/C/Alloc.h +++ b/C/Alloc.h @@ -1,5 +1,5 @@ /* Alloc.h -- Memory allocation functions -2023-03-04 : Igor Pavlov : Public domain */ +2024-01-22 : Igor Pavlov : Public domain */ #ifndef ZIP7_INC_ALLOC_H #define ZIP7_INC_ALLOC_H @@ -22,6 +22,9 @@ void *MyAlloc(size_t size); void MyFree(void *address); void *MyRealloc(void *address, size_t size); +void *z7_AlignedAlloc(size_t size); +void z7_AlignedFree(void *p); + #ifdef _WIN32 #ifdef Z7_LARGE_PAGES @@ -33,12 +36,14 @@ void MidFree(void *address); void *BigAlloc(size_t size); void BigFree(void *address); +/* #define Z7_BIG_ALLOC_IS_ZERO_FILLED */ + #else -#define MidAlloc(size) MyAlloc(size) -#define MidFree(address) MyFree(address) -#define BigAlloc(size) MyAlloc(size) -#define BigFree(address) MyFree(address) +#define MidAlloc(size) z7_AlignedAlloc(size) +#define MidFree(address) z7_AlignedFree(address) +#define BigAlloc(size) z7_AlignedAlloc(size) +#define BigFree(address) z7_AlignedFree(address) #endif diff --git a/C/Asm_c.mak b/C/Asm_c.mak new file mode 100644 index 0000000..9431816 --- /dev/null +++ b/C/Asm_c.mak @@ -0,0 +1,12 @@ +!IFDEF ASM_OBJS +!IF "$(PLATFORM)" == "arm64" +$(ASM_OBJS): ../../../Asm/arm64/$(*B).S + $(COMPL_ASM_CLANG) +!ELSEIF "$(PLATFORM)" == "arm" +$(ASM_OBJS): ../../../Asm/arm/$(*B).asm + $(COMPL_ASM) +!ELSEIF "$(PLATFORM)" != "ia64" && "$(PLATFORM)" != "mips" +$(ASM_OBJS): ../../../Asm/x86/$(*B).asm + $(COMPL_ASM) +!ENDIF +!ENDIF diff --git a/C/Blake2.h b/C/Blake2.h index 7235235..801ea7a 100644 --- a/C/Blake2.h +++ b/C/Blake2.h @@ -1,47 +1,104 @@ -/* Blake2.h -- BLAKE2 Hash -2023-03-04 : Igor Pavlov : Public domain -2015 : Samuel Neves : Public domain */ +/* Blake2.h -- BLAKE2sp Hash +2024-01-17 : Igor Pavlov : Public domain */ #ifndef ZIP7_INC_BLAKE2_H #define ZIP7_INC_BLAKE2_H #include "7zTypes.h" -EXTERN_C_BEGIN +#if 0 +#include "Compiler.h" +#include "CpuArch.h" +#if defined(MY_CPU_X86_OR_AMD64) +#if defined(__SSE2__) \ + || defined(_MSC_VER) && _MSC_VER > 1200 \ + || defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 30300) \ + || defined(__clang__) \ + || defined(__INTEL_COMPILER) +#include // SSE2 +#endif -#define BLAKE2S_BLOCK_SIZE 64 -#define BLAKE2S_DIGEST_SIZE 32 -#define BLAKE2SP_PARALLEL_DEGREE 8 +#if defined(__AVX2__) \ + || defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 40900) \ + || defined(Z7_APPLE_CLANG_VERSION) && (Z7_APPLE_CLANG_VERSION >= 40600) \ + || defined(Z7_LLVM_CLANG_VERSION) && (Z7_LLVM_CLANG_VERSION >= 30100) \ + || defined(Z7_MSC_VER_ORIGINAL) && (Z7_MSC_VER_ORIGINAL >= 1800) \ + || defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1400) +#include +#if defined(__clang__) +#include +#include +#endif +#endif // avx2 +#endif // MY_CPU_X86_OR_AMD64 +#endif // 0 -typedef struct -{ - UInt32 h[8]; - UInt32 t[2]; - UInt32 f[2]; - Byte buf[BLAKE2S_BLOCK_SIZE]; - UInt32 bufPos; - UInt32 lastNode_f1; - UInt32 dummy[2]; /* for sizeof(CBlake2s) alignment */ -} CBlake2s; - -/* You need to xor CBlake2s::h[i] with input parameter block after Blake2s_Init0() */ -/* -void Blake2s_Init0(CBlake2s *p); -void Blake2s_Update(CBlake2s *p, const Byte *data, size_t size); -void Blake2s_Final(CBlake2s *p, Byte *digest); -*/ +EXTERN_C_BEGIN +#define Z7_BLAKE2S_BLOCK_SIZE 64 +#define Z7_BLAKE2S_DIGEST_SIZE 32 +#define Z7_BLAKE2SP_PARALLEL_DEGREE 8 +#define Z7_BLAKE2SP_NUM_STRUCT_WORDS 16 +#if 1 || defined(Z7_BLAKE2SP_USE_FUNCTIONS) +typedef void (Z7_FASTCALL *Z7_BLAKE2SP_FUNC_COMPRESS)(UInt32 *states, const Byte *data, const Byte *end); +typedef void (Z7_FASTCALL *Z7_BLAKE2SP_FUNC_INIT)(UInt32 *states); +#endif + +// it's required that CBlake2sp is aligned for 32-bytes, +// because the code can use unaligned access with sse and avx256. +// but 64-bytes alignment can be better. +MY_ALIGN(64) typedef struct { - CBlake2s S[BLAKE2SP_PARALLEL_DEGREE]; - unsigned bufPos; -} CBlake2sp; + union + { +#if 0 +#if defined(MY_CPU_X86_OR_AMD64) +#if defined(__SSE2__) \ + || defined(_MSC_VER) && _MSC_VER > 1200 \ + || defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 30300) \ + || defined(__clang__) \ + || defined(__INTEL_COMPILER) + __m128i _pad_align_128bit[4]; +#endif // sse2 +#if defined(__AVX2__) \ + || defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 40900) \ + || defined(Z7_APPLE_CLANG_VERSION) && (Z7_APPLE_CLANG_VERSION >= 40600) \ + || defined(Z7_LLVM_CLANG_VERSION) && (Z7_LLVM_CLANG_VERSION >= 30100) \ + || defined(Z7_MSC_VER_ORIGINAL) && (Z7_MSC_VER_ORIGINAL >= 1800) \ + || defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1400) + __m256i _pad_align_256bit[2]; +#endif // avx2 +#endif // x86 +#endif // 0 + void * _pad_align_ptr[8]; + UInt32 _pad_align_32bit[16]; + struct + { + unsigned cycPos; + unsigned _pad_unused; +#if 1 || defined(Z7_BLAKE2SP_USE_FUNCTIONS) + Z7_BLAKE2SP_FUNC_COMPRESS func_Compress_Fast; + Z7_BLAKE2SP_FUNC_COMPRESS func_Compress_Single; + Z7_BLAKE2SP_FUNC_INIT func_Init; + Z7_BLAKE2SP_FUNC_INIT func_Final; +#endif + } header; + } u; + // MY_ALIGN(64) + UInt32 states[Z7_BLAKE2SP_PARALLEL_DEGREE * Z7_BLAKE2SP_NUM_STRUCT_WORDS]; + // MY_ALIGN(64) + UInt32 buf32[Z7_BLAKE2SP_PARALLEL_DEGREE * Z7_BLAKE2SP_NUM_STRUCT_WORDS * 2]; +} CBlake2sp; +BoolInt Blake2sp_SetFunction(CBlake2sp *p, unsigned algo); void Blake2sp_Init(CBlake2sp *p); +void Blake2sp_InitState(CBlake2sp *p); void Blake2sp_Update(CBlake2sp *p, const Byte *data, size_t size); void Blake2sp_Final(CBlake2sp *p, Byte *digest); +void z7_Black2sp_Prepare(void); EXTERN_C_END diff --git a/C/Blake2s.c b/C/Blake2s.c index 2a84b57..459e76b 100644 --- a/C/Blake2s.c +++ b/C/Blake2s.c @@ -1,250 +1,2645 @@ -/* Blake2s.c -- BLAKE2s and BLAKE2sp Hash -2023-03-04 : Igor Pavlov : Public domain -2015 : Samuel Neves : Public domain */ +/* Blake2s.c -- BLAKE2sp Hash +2024-01-29 : Igor Pavlov : Public domain +2015-2019 : Samuel Neves : original code : CC0 1.0 Universal (CC0 1.0). */ #include "Precomp.h" +// #include #include #include "Blake2.h" -#include "CpuArch.h" #include "RotateDefs.h" +#include "Compiler.h" +#include "CpuArch.h" + +#if defined(__SSE2__) + #define Z7_BLAKE2S_USE_VECTORS +#elif defined(MY_CPU_X86_OR_AMD64) + #if defined(_MSC_VER) && _MSC_VER > 1200 \ + || defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 30300) \ + || defined(__clang__) \ + || defined(__INTEL_COMPILER) + #define Z7_BLAKE2S_USE_VECTORS + #endif +#endif + +#ifdef Z7_BLAKE2S_USE_VECTORS + +#define Z7_BLAKE2SP_USE_FUNCTIONS + +// define Z7_BLAKE2SP_STRUCT_IS_NOT_ALIGNED, if CBlake2sp can be non aligned for 32-bytes. +// #define Z7_BLAKE2SP_STRUCT_IS_NOT_ALIGNED + +// SSSE3 : for _mm_shuffle_epi8 (pshufb) that improves the performance for 5-15%. +#if defined(__SSSE3__) + #define Z7_BLAKE2S_USE_SSSE3 +#elif defined(Z7_MSC_VER_ORIGINAL) && (Z7_MSC_VER_ORIGINAL >= 1500) \ + || defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 40300) \ + || defined(Z7_APPLE_CLANG_VERSION) && (Z7_APPLE_CLANG_VERSION >= 40000) \ + || defined(Z7_LLVM_CLANG_VERSION) && (Z7_LLVM_CLANG_VERSION >= 20300) \ + || defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1000) + #define Z7_BLAKE2S_USE_SSSE3 +#endif + +#ifdef Z7_BLAKE2S_USE_SSSE3 +/* SSE41 : for _mm_insert_epi32 (pinsrd) + it can slightly reduce code size and improves the performance in some cases. + it's used only for last 512-1024 bytes, if FAST versions (2 or 3) of vector algos are used. + it can be used for all blocks in another algos (4+). +*/ +#if defined(__SSE4_1__) + #define Z7_BLAKE2S_USE_SSE41 +#elif defined(Z7_MSC_VER_ORIGINAL) && (Z7_MSC_VER_ORIGINAL >= 1500) \ + || defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 40300) \ + || defined(Z7_APPLE_CLANG_VERSION) && (Z7_APPLE_CLANG_VERSION >= 40000) \ + || defined(Z7_LLVM_CLANG_VERSION) && (Z7_LLVM_CLANG_VERSION >= 20300) \ + || defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1000) + #define Z7_BLAKE2S_USE_SSE41 +#endif +#endif // SSSE3 + +#if defined(__GNUC__) || defined(__clang__) + #if defined(Z7_BLAKE2S_USE_SSE41) + #define BLAKE2S_ATTRIB_128BIT __attribute__((__target__("sse4.1"))) + #elif defined(Z7_BLAKE2S_USE_SSSE3) + #define BLAKE2S_ATTRIB_128BIT __attribute__((__target__("ssse3"))) + #else + #define BLAKE2S_ATTRIB_128BIT __attribute__((__target__("sse2"))) + #endif +#endif + + +#if defined(__AVX2__) + #define Z7_BLAKE2S_USE_AVX2 +#else + #if defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 40900) \ + || defined(Z7_APPLE_CLANG_VERSION) && (Z7_APPLE_CLANG_VERSION >= 40600) \ + || defined(Z7_LLVM_CLANG_VERSION) && (Z7_LLVM_CLANG_VERSION >= 30100) + #define Z7_BLAKE2S_USE_AVX2 + #ifdef Z7_BLAKE2S_USE_AVX2 + #define BLAKE2S_ATTRIB_AVX2 __attribute__((__target__("avx2"))) + #endif + #elif defined(Z7_MSC_VER_ORIGINAL) && (Z7_MSC_VER_ORIGINAL >= 1800) \ + || defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1400) + #if (Z7_MSC_VER_ORIGINAL == 1900) + #pragma warning(disable : 4752) // found Intel(R) Advanced Vector Extensions; consider using /arch:AVX + #endif + #define Z7_BLAKE2S_USE_AVX2 + #endif +#endif + +#ifdef Z7_BLAKE2S_USE_SSE41 +#include // SSE4.1 +#elif defined(Z7_BLAKE2S_USE_SSSE3) +#include // SSSE3 +#else +#include // SSE2 +#endif + +#ifdef Z7_BLAKE2S_USE_AVX2 +#include +#if defined(__clang__) +#include +#include +#endif +#endif // avx2 + + +#if defined(__AVX512F__) && defined(__AVX512VL__) + // && defined(Z7_MSC_VER_ORIGINAL) && (Z7_MSC_VER_ORIGINAL > 1930) + #define Z7_BLAKE2S_USE_AVX512_ALWAYS + // #pragma message ("=== Blake2s AVX512") +#endif -#define rotr32 rotrFixed -#define BLAKE2S_NUM_ROUNDS 10 -#define BLAKE2S_FINAL_FLAG (~(UInt32)0) +#define Z7_BLAKE2S_USE_V128_FAST +// for speed optimization for small messages: +// #define Z7_BLAKE2S_USE_V128_WAY2 +#ifdef Z7_BLAKE2S_USE_AVX2 + +// for debug: +// gather is slow +// #define Z7_BLAKE2S_USE_GATHER + + #define Z7_BLAKE2S_USE_AVX2_FAST +// for speed optimization for small messages: +// #define Z7_BLAKE2S_USE_AVX2_WAY2 +// #define Z7_BLAKE2S_USE_AVX2_WAY4 +#if defined(Z7_BLAKE2S_USE_AVX2_WAY2) || \ + defined(Z7_BLAKE2S_USE_AVX2_WAY4) + #define Z7_BLAKE2S_USE_AVX2_WAY_SLOW +#endif +#endif + + #define Z7_BLAKE2SP_ALGO_DEFAULT 0 + #define Z7_BLAKE2SP_ALGO_SCALAR 1 +#ifdef Z7_BLAKE2S_USE_V128_FAST + #define Z7_BLAKE2SP_ALGO_V128_FAST 2 +#endif +#ifdef Z7_BLAKE2S_USE_AVX2_FAST + #define Z7_BLAKE2SP_ALGO_V256_FAST 3 +#endif + #define Z7_BLAKE2SP_ALGO_V128_WAY1 4 +#ifdef Z7_BLAKE2S_USE_V128_WAY2 + #define Z7_BLAKE2SP_ALGO_V128_WAY2 5 +#endif +#ifdef Z7_BLAKE2S_USE_AVX2_WAY2 + #define Z7_BLAKE2SP_ALGO_V256_WAY2 6 +#endif +#ifdef Z7_BLAKE2S_USE_AVX2_WAY4 + #define Z7_BLAKE2SP_ALGO_V256_WAY4 7 +#endif + +#endif // Z7_BLAKE2S_USE_VECTORS + + + + +#define BLAKE2S_FINAL_FLAG (~(UInt32)0) +#define NSW Z7_BLAKE2SP_NUM_STRUCT_WORDS +#define SUPER_BLOCK_SIZE (Z7_BLAKE2S_BLOCK_SIZE * Z7_BLAKE2SP_PARALLEL_DEGREE) +#define SUPER_BLOCK_MASK (SUPER_BLOCK_SIZE - 1) + +#define V_INDEX_0_0 0 +#define V_INDEX_1_0 1 +#define V_INDEX_2_0 2 +#define V_INDEX_3_0 3 +#define V_INDEX_0_1 4 +#define V_INDEX_1_1 5 +#define V_INDEX_2_1 6 +#define V_INDEX_3_1 7 +#define V_INDEX_0_2 8 +#define V_INDEX_1_2 9 +#define V_INDEX_2_2 10 +#define V_INDEX_3_2 11 +#define V_INDEX_0_3 12 +#define V_INDEX_1_3 13 +#define V_INDEX_2_3 14 +#define V_INDEX_3_3 15 +#define V_INDEX_4_0 0 +#define V_INDEX_5_0 1 +#define V_INDEX_6_0 2 +#define V_INDEX_7_0 3 +#define V_INDEX_7_1 4 +#define V_INDEX_4_1 5 +#define V_INDEX_5_1 6 +#define V_INDEX_6_1 7 +#define V_INDEX_6_2 8 +#define V_INDEX_7_2 9 +#define V_INDEX_4_2 10 +#define V_INDEX_5_2 11 +#define V_INDEX_5_3 12 +#define V_INDEX_6_3 13 +#define V_INDEX_7_3 14 +#define V_INDEX_4_3 15 + +#define V(row, col) v[V_INDEX_ ## row ## _ ## col] + +#define k_Blake2s_IV_0 0x6A09E667UL +#define k_Blake2s_IV_1 0xBB67AE85UL +#define k_Blake2s_IV_2 0x3C6EF372UL +#define k_Blake2s_IV_3 0xA54FF53AUL +#define k_Blake2s_IV_4 0x510E527FUL +#define k_Blake2s_IV_5 0x9B05688CUL +#define k_Blake2s_IV_6 0x1F83D9ABUL +#define k_Blake2s_IV_7 0x5BE0CD19UL + +#define KIV(n) (k_Blake2s_IV_## n) + +#ifdef Z7_BLAKE2S_USE_VECTORS +MY_ALIGN(16) static const UInt32 k_Blake2s_IV[8] = { - 0x6A09E667UL, 0xBB67AE85UL, 0x3C6EF372UL, 0xA54FF53AUL, - 0x510E527FUL, 0x9B05688CUL, 0x1F83D9ABUL, 0x5BE0CD19UL + KIV(0), KIV(1), KIV(2), KIV(3), KIV(4), KIV(5), KIV(6), KIV(7) }; +#endif -static const Byte k_Blake2s_Sigma[BLAKE2S_NUM_ROUNDS][16] = -{ - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } , - { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } , - { 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 } , - { 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 } , - { 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 } , - { 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 } , - { 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 } , - { 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 } , - { 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 } , - { 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13 , 0 } , -}; +#define STATE_T(s) ((s) + 8) +#define STATE_F(s) ((s) + 10) + +#ifdef Z7_BLAKE2S_USE_VECTORS +#define LOAD_128(p) _mm_load_si128 ((const __m128i *)(const void *)(p)) +#define LOADU_128(p) _mm_loadu_si128((const __m128i *)(const void *)(p)) +#ifdef Z7_BLAKE2SP_STRUCT_IS_NOT_ALIGNED + // here we use unaligned load and stores + // use this branch if CBlake2sp can be unaligned for 16 bytes + #define STOREU_128(p, r) _mm_storeu_si128((__m128i *)(void *)(p), r) + #define LOAD_128_FROM_STRUCT(p) LOADU_128(p) + #define STORE_128_TO_STRUCT(p, r) STOREU_128(p, r) +#else + // here we use aligned load and stores + // use this branch if CBlake2sp is aligned for 16 bytes + #define STORE_128(p, r) _mm_store_si128((__m128i *)(void *)(p), r) + #define LOAD_128_FROM_STRUCT(p) LOAD_128(p) + #define STORE_128_TO_STRUCT(p, r) STORE_128(p, r) +#endif -static void Blake2s_Init0(CBlake2s *p) +#endif // Z7_BLAKE2S_USE_VECTORS + + +#if 0 +static void PrintState(const UInt32 *s, unsigned num) +{ + unsigned i; + printf("\n"); + for (i = 0; i < num; i++) + printf(" %08x", (unsigned)s[i]); +} +static void PrintStates2(const UInt32 *s, unsigned x, unsigned y) { unsigned i; - for (i = 0; i < 8; i++) - p->h[i] = k_Blake2s_IV[i]; - p->t[0] = 0; - p->t[1] = 0; - p->f[0] = 0; - p->f[1] = 0; - p->bufPos = 0; - p->lastNode_f1 = 0; + for (i = 0; i < y; i++) + PrintState(s + i * x, x); + printf("\n"); } +#endif + + +#define REP8_MACRO(m) { m(0) m(1) m(2) m(3) m(4) m(5) m(6) m(7) } + +#define BLAKE2S_NUM_ROUNDS 10 + +#if defined(Z7_BLAKE2S_USE_VECTORS) +#define ROUNDS_LOOP(mac) \ + { unsigned r; for (r = 0; r < BLAKE2S_NUM_ROUNDS; r++) mac(r) } +#endif +/* +#define ROUNDS_LOOP_2(mac) \ + { unsigned r; for (r = 0; r < BLAKE2S_NUM_ROUNDS; r += 2) { mac(r) mac(r + 1) } } +*/ +#if 0 || 1 && !defined(Z7_BLAKE2S_USE_VECTORS) +#define ROUNDS_LOOP_UNROLLED(m) \ + { m(0) m(1) m(2) m(3) m(4) m(5) m(6) m(7) m(8) m(9) } +#endif + +#define SIGMA_TABLE(M) \ + M( 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 ), \ + M( 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 ), \ + M( 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 ), \ + M( 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 ), \ + M( 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 ), \ + M( 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 ), \ + M( 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 ), \ + M( 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 ), \ + M( 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 ), \ + M( 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0 ) + +#define SIGMA_TABLE_MULT(m, a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15) \ + { a0*m,a1*m,a2*m,a3*m,a4*m,a5*m,a6*m,a7*m,a8*m,a9*m,a10*m,a11*m,a12*m,a13*m,a14*m,a15*m } +#define SIGMA_TABLE_MULT_4( a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15) \ + SIGMA_TABLE_MULT(4, a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15) + +// MY_ALIGN(32) +MY_ALIGN(16) +static const Byte k_Blake2s_Sigma_4[BLAKE2S_NUM_ROUNDS][16] = + { SIGMA_TABLE(SIGMA_TABLE_MULT_4) }; + +#define GET_SIGMA_PTR(p, index) \ + ((const void *)((const Byte *)(const void *)(p) + (index))) +#define GET_STATE_TABLE_PTR_FROM_BYTE_POS(s, pos) \ + ((UInt32 *)(void *)((Byte *)(void *)(s) + (pos))) -static void Blake2s_Compress(CBlake2s *p) + +#ifdef Z7_BLAKE2S_USE_VECTORS + + +#if 0 + // use loading constants from memory + // is faster for some compilers. + #define KK4(n) KIV(n), KIV(n), KIV(n), KIV(n) +MY_ALIGN(64) +static const UInt32 k_Blake2s_IV_WAY4[]= { - UInt32 m[16]; - UInt32 v[16]; - + KK4(0), KK4(1), KK4(2), KK4(3), KK4(4), KK4(5), KK4(6), KK4(7) +}; + #define GET_128_IV_WAY4(i) LOAD_128(k_Blake2s_IV_WAY4 + 4 * (i)) +#else + // use constant generation: + #define GET_128_IV_WAY4(i) _mm_set1_epi32((Int32)KIV(i)) +#endif + + +#ifdef Z7_BLAKE2S_USE_AVX2_WAY_SLOW +#define GET_CONST_128_FROM_ARRAY32(k) \ + _mm_set_epi32((Int32)(k)[3], (Int32)(k)[2], (Int32)(k)[1], (Int32)(k)[0]) +#endif + + +#if 0 +#define k_r8 _mm_set_epi8(12, 15, 14, 13, 8, 11, 10, 9, 4, 7, 6, 5, 0, 3, 2, 1) +#define k_r16 _mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2) +#define k_inc _mm_set_epi32(0, 0, 0, Z7_BLAKE2S_BLOCK_SIZE) +#define k_iv0_128 GET_CONST_128_FROM_ARRAY32(k_Blake2s_IV + 0) +#define k_iv4_128 GET_CONST_128_FROM_ARRAY32(k_Blake2s_IV + 4) +#else +#if defined(Z7_BLAKE2S_USE_SSSE3) && \ + !defined(Z7_BLAKE2S_USE_AVX512_ALWAYS) +MY_ALIGN(16) static const Byte k_r8_arr [16] = { 1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8 ,13, 14, 15, 12 }; +MY_ALIGN(16) static const Byte k_r16_arr[16] = { 2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13 }; +#define k_r8 LOAD_128(k_r8_arr) +#define k_r16 LOAD_128(k_r16_arr) +#endif +MY_ALIGN(16) static const UInt32 k_inc_arr[4] = { Z7_BLAKE2S_BLOCK_SIZE, 0, 0, 0 }; +#define k_inc LOAD_128(k_inc_arr) +#define k_iv0_128 LOAD_128(k_Blake2s_IV + 0) +#define k_iv4_128 LOAD_128(k_Blake2s_IV + 4) +#endif + + +#ifdef Z7_BLAKE2S_USE_AVX2_WAY_SLOW + +#ifdef Z7_BLAKE2S_USE_AVX2 +#if defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION < 80000) + #define MY_mm256_set_m128i(hi, lo) _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1) +#else + #define MY_mm256_set_m128i _mm256_set_m128i +#endif + +#define SET_FROM_128(a) MY_mm256_set_m128i(a, a) + +#ifndef Z7_BLAKE2S_USE_AVX512_ALWAYS +MY_ALIGN(32) static const Byte k_r8_arr_256 [32] = +{ + 1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8 ,13, 14, 15, 12, + 1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8 ,13, 14, 15, 12 +}; +MY_ALIGN(32) static const Byte k_r16_arr_256[32] = +{ + 2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13, + 2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13 +}; +#define k_r8_256 LOAD_256(k_r8_arr_256) +#define k_r16_256 LOAD_256(k_r16_arr_256) +#endif + +// #define k_r8_256 SET_FROM_128(_mm_set_epi8(12, 15, 14, 13, 8, 11, 10, 9, 4, 7, 6, 5, 0, 3, 2, 1)) +// #define k_r16_256 SET_FROM_128(_mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2)) +// #define k_inc_256 SET_FROM_128(_mm_set_epi32(0, 0, 0, Z7_BLAKE2S_BLOCK_SIZE)) +// #define k_iv0_256 SET_FROM_128(GET_CONST_128_FROM_ARRAY32(k_Blake2s_IV + 0)) +#define k_iv4_256 SET_FROM_128(GET_CONST_128_FROM_ARRAY32(k_Blake2s_IV + 4)) +#endif // Z7_BLAKE2S_USE_AVX2_WAY_SLOW +#endif + + +/* +IPC(TP) ports: +1 p__5 : skl- : SSE : shufps : _mm_shuffle_ps +2 p_15 : icl+ +1 p__5 : nhm-bdw : SSE : xorps : _mm_xor_ps +3 p015 : skl+ + +3 p015 : SSE2 : pxor : _mm_xor_si128 +2 p_15: snb-bdw : SSE2 : padd : _mm_add_epi32 +2 p0_5: mrm-wsm : +3 p015 : skl+ + +2 p_15 : ivb-,icl+ : SSE2 : punpcklqdq, punpckhqdq, punpckldq, punpckhdq +2 p_15 : : SSE2 : pshufd : _mm_shuffle_epi32 +2 p_15 : : SSE2 : pshuflw : _mm_shufflelo_epi16 +2 p_15 : : SSE2 : psrldq : +2 p_15 : : SSE3 : pshufb : _mm_shuffle_epi8 +2 p_15 : : SSE4 : pblendw : _mm_blend_epi16 +1 p__5 : hsw-skl : * + +1 p0 : SSE2 : pslld (i8) : _mm_slli_si128 +2 p01 : skl+ : + +2 p_15 : ivb- : SSE3 : palignr +1 p__5 : hsw+ + +2 p_15 + p23 : ivb-, icl+ : SSE4 : pinsrd : _mm_insert_epi32(xmm, m32, i8) +1 p__5 + p23 : hsw-skl +1 p_15 + p5 : ivb-, ice+ : SSE4 : pinsrd : _mm_insert_epi32(xmm, r32, i8) +0.5 2*p5 : hsw-skl + +2 p23 : SSE2 : movd (m32) +3 p23A : adl : +1 p5: : SSE2 : movd (r32) +*/ + +#if 0 && defined(__XOP__) +// we must debug and test __XOP__ instruction +#include +#include + #define LOAD_ROTATE_CONSTS + #define MM_ROR_EPI32(r, c) _mm_roti_epi32(r, -(c)) + #define Z7_BLAKE2S_MM_ROR_EPI32_IS_SUPPORTED +#elif 1 && defined(Z7_BLAKE2S_USE_AVX512_ALWAYS) + #define LOAD_ROTATE_CONSTS + #define MM_ROR_EPI32(r, c) _mm_ror_epi32(r, c) + #define Z7_BLAKE2S_MM_ROR_EPI32_IS_SUPPORTED +#else + +// MSVC_1937+ uses "orps" instruction for _mm_or_si128(). +// But "orps" has low throughput: TP=1 for bdw-nhm. +// So it can be better to use _mm_add_epi32()/"paddd" (TP=2 for bdw-nhm) instead of "xorps". +// But "orps" is fast for modern cpus (skl+). +// So we are default with "or" version: +#if 0 || 0 && defined(Z7_MSC_VER_ORIGINAL) && Z7_MSC_VER_ORIGINAL > 1937 + // minor optimization for some old cpus, if "xorps" is slow. + #define MM128_EPI32_OR_or_ADD _mm_add_epi32 +#else + #define MM128_EPI32_OR_or_ADD _mm_or_si128 +#endif + + #define MM_ROR_EPI32_VIA_SHIFT(r, c)( \ + MM128_EPI32_OR_or_ADD( \ + _mm_srli_epi32((r), (c)), \ + _mm_slli_epi32((r), 32-(c)))) + #if defined(Z7_BLAKE2S_USE_SSSE3) || defined(Z7_BLAKE2S_USE_SSE41) + #define LOAD_ROTATE_CONSTS \ + const __m128i r8 = k_r8; \ + const __m128i r16 = k_r16; + #define MM_ROR_EPI32(r, c) ( \ + ( 8==(c)) ? _mm_shuffle_epi8(r,r8) \ + : (16==(c)) ? _mm_shuffle_epi8(r,r16) \ + : MM_ROR_EPI32_VIA_SHIFT(r, c)) + #else + #define LOAD_ROTATE_CONSTS + #define MM_ROR_EPI32(r, c) ( \ + (16==(c)) ? _mm_shufflehi_epi16(_mm_shufflelo_epi16(r, 0xb1), 0xb1) \ + : MM_ROR_EPI32_VIA_SHIFT(r, c)) + #endif +#endif + +/* +we have 3 main ways to load 4 32-bit integers to __m128i: + 1) SSE2: _mm_set_epi32() + 2) SSE2: _mm_unpacklo_epi64() / _mm_unpacklo_epi32 / _mm_cvtsi32_si128() + 3) SSE41: _mm_insert_epi32() and _mm_cvtsi32_si128() +good compiler for _mm_set_epi32() generates these instructions: +{ + movd xmm, [m32]; vpunpckldq; vpunpckldq; vpunpcklqdq; +} +good new compiler generates one instruction +{ + for _mm_insert_epi32() : { pinsrd xmm, [m32], i } + for _mm_cvtsi32_si128() : { movd xmm, [m32] } +} +but vc2010 generates slow pair of instructions: +{ + for _mm_insert_epi32() : { mov r32, [m32]; pinsrd xmm, r32, i } + for _mm_cvtsi32_si128() : { mov r32, [m32]; movd xmm, r32 } +} +_mm_insert_epi32() (pinsrd) code reduces xmm register pressure +in comparison with _mm_set_epi32() (movd + vpunpckld) code. +Note that variant with "movd xmm, r32" can be more slow, +but register pressure can be more important. +So we can force to "pinsrd" always. +*/ +// #if !defined(Z7_MSC_VER_ORIGINAL) || Z7_MSC_VER_ORIGINAL > 1600 || defined(MY_CPU_X86) +#ifdef Z7_BLAKE2S_USE_SSE41 + /* _mm_set_epi32() can be more effective for GCC and CLANG + _mm_insert_epi32() is more effective for MSVC */ + #if 0 || 1 && defined(Z7_MSC_VER_ORIGINAL) + #define Z7_BLAKE2S_USE_INSERT_INSTRUCTION + #endif +#endif // USE_SSE41 +// #endif + +#ifdef Z7_BLAKE2S_USE_INSERT_INSTRUCTION + // for SSE4.1 +#define MM_LOAD_EPI32_FROM_4_POINTERS(p0, p1, p2, p3) \ + _mm_insert_epi32( \ + _mm_insert_epi32( \ + _mm_insert_epi32( \ + _mm_cvtsi32_si128( \ + *(const Int32 *)p0), \ + *(const Int32 *)p1, 1), \ + *(const Int32 *)p2, 2), \ + *(const Int32 *)p3, 3) +#elif 0 || 1 && defined(Z7_MSC_VER_ORIGINAL) +/* MSVC 1400 implements _mm_set_epi32() via slow memory write/read. + Also _mm_unpacklo_epi32 is more effective for another MSVC compilers. + But _mm_set_epi32() is more effective for GCC and CLANG. + So we use _mm_unpacklo_epi32 for MSVC only */ +#define MM_LOAD_EPI32_FROM_4_POINTERS(p0, p1, p2, p3) \ + _mm_unpacklo_epi64( \ + _mm_unpacklo_epi32( _mm_cvtsi32_si128(*(const Int32 *)p0), \ + _mm_cvtsi32_si128(*(const Int32 *)p1)), \ + _mm_unpacklo_epi32( _mm_cvtsi32_si128(*(const Int32 *)p2), \ + _mm_cvtsi32_si128(*(const Int32 *)p3))) +#else +#define MM_LOAD_EPI32_FROM_4_POINTERS(p0, p1, p2, p3) \ + _mm_set_epi32( \ + *(const Int32 *)p3, \ + *(const Int32 *)p2, \ + *(const Int32 *)p1, \ + *(const Int32 *)p0) +#endif + +#define SET_ROW_FROM_SIGMA_BASE(input, i0, i1, i2, i3) \ + MM_LOAD_EPI32_FROM_4_POINTERS( \ + GET_SIGMA_PTR(input, i0), \ + GET_SIGMA_PTR(input, i1), \ + GET_SIGMA_PTR(input, i2), \ + GET_SIGMA_PTR(input, i3)) + +#define SET_ROW_FROM_SIGMA(input, sigma_index) \ + SET_ROW_FROM_SIGMA_BASE(input, \ + sigma[(sigma_index) ], \ + sigma[(sigma_index) + 2 * 1], \ + sigma[(sigma_index) + 2 * 2], \ + sigma[(sigma_index) + 2 * 3]) \ + + +#define ADD_128(a, b) _mm_add_epi32(a, b) +#define XOR_128(a, b) _mm_xor_si128(a, b) + +#define D_ADD_128(dest, src) dest = ADD_128(dest, src) +#define D_XOR_128(dest, src) dest = XOR_128(dest, src) +#define D_ROR_128(dest, shift) dest = MM_ROR_EPI32(dest, shift) +#define D_ADD_EPI64_128(dest, src) dest = _mm_add_epi64(dest, src) + + +#define AXR(a, b, d, shift) \ + D_ADD_128(a, b); \ + D_XOR_128(d, a); \ + D_ROR_128(d, shift); + +#define AXR2(a, b, c, d, input, sigma_index, shift1, shift2) \ + a = _mm_add_epi32 (a, SET_ROW_FROM_SIGMA(input, sigma_index)); \ + AXR(a, b, d, shift1) \ + AXR(c, d, b, shift2) + +#define ROTATE_WORDS_TO_RIGHT(a, n) \ + a = _mm_shuffle_epi32(a, _MM_SHUFFLE((3+n)&3, (2+n)&3, (1+n)&3, (0+n)&3)); + +#define AXR4(a, b, c, d, input, sigma_index) \ + AXR2(a, b, c, d, input, sigma_index, 16, 12) \ + AXR2(a, b, c, d, input, sigma_index + 1, 8, 7) \ + +#define RR2(a, b, c, d, input) \ + { \ + AXR4(a, b, c, d, input, 0) \ + ROTATE_WORDS_TO_RIGHT(b, 1) \ + ROTATE_WORDS_TO_RIGHT(c, 2) \ + ROTATE_WORDS_TO_RIGHT(d, 3) \ + AXR4(a, b, c, d, input, 8) \ + ROTATE_WORDS_TO_RIGHT(b, 3) \ + ROTATE_WORDS_TO_RIGHT(c, 2) \ + ROTATE_WORDS_TO_RIGHT(d, 1) \ + } + + +/* +Way1: +per 64 bytes block: +10 rounds * 4 iters * (7 + 2) = 360 cycles = if pslld TP=1 + * (7 + 1) = 320 cycles = if pslld TP=2 (skl+) +additional operations per 7_op_iter : +4 movzx byte mem +1 movd mem +3 pinsrd mem +1.5 pshufd +*/ + +static +#if 0 || 0 && (defined(Z7_BLAKE2S_USE_V128_WAY2) || \ + defined(Z7_BLAKE2S_USE_V256_WAY2)) + Z7_NO_INLINE +#else + Z7_FORCE_INLINE +#endif +#ifdef BLAKE2S_ATTRIB_128BIT + BLAKE2S_ATTRIB_128BIT +#endif +void +Z7_FASTCALL +Blake2s_Compress_V128_Way1(UInt32 * const s, const Byte * const input) +{ + __m128i a, b, c, d; + __m128i f0, f1; + + LOAD_ROTATE_CONSTS + d = LOAD_128_FROM_STRUCT(STATE_T(s)); + c = k_iv0_128; + a = f0 = LOAD_128_FROM_STRUCT(s); + b = f1 = LOAD_128_FROM_STRUCT(s + 4); + D_ADD_EPI64_128(d, k_inc); + STORE_128_TO_STRUCT (STATE_T(s), d); + D_XOR_128(d, k_iv4_128); + +#define RR(r) { const Byte * const sigma = k_Blake2s_Sigma_4[r]; \ + RR2(a, b, c, d, input) } + + ROUNDS_LOOP(RR) +#undef RR + + STORE_128_TO_STRUCT(s , XOR_128(f0, XOR_128(a, c))); + STORE_128_TO_STRUCT(s + 4, XOR_128(f1, XOR_128(b, d))); +} + + +static +Z7_NO_INLINE +#ifdef BLAKE2S_ATTRIB_128BIT + BLAKE2S_ATTRIB_128BIT +#endif +void +Z7_FASTCALL +Blake2sp_Compress2_V128_Way1(UInt32 *s_items, const Byte *data, const Byte *end) +{ + size_t pos = 0; + do { - unsigned i; + UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(s_items, pos); + Blake2s_Compress_V128_Way1(s, data); + data += Z7_BLAKE2S_BLOCK_SIZE; + pos += Z7_BLAKE2S_BLOCK_SIZE; + pos &= SUPER_BLOCK_MASK; + } + while (data != end); +} + + +#if defined(Z7_BLAKE2S_USE_V128_WAY2) || \ + defined(Z7_BLAKE2S_USE_AVX2_WAY2) +#if 1 + #define Z7_BLAKE2S_CompressSingleBlock(s, data) \ + Blake2sp_Compress2_V128_Way1(s, data, \ + (const Byte *)(const void *)(data) + Z7_BLAKE2S_BLOCK_SIZE) +#else + #define Z7_BLAKE2S_CompressSingleBlock Blake2s_Compress_V128_Way1 +#endif +#endif + + +#if (defined(Z7_BLAKE2S_USE_AVX2_WAY_SLOW) || \ + defined(Z7_BLAKE2S_USE_V128_WAY2)) && \ + !defined(Z7_BLAKE2S_USE_GATHER) +#define AXR2_LOAD_INDEXES(sigma_index) \ + const unsigned i0 = sigma[(sigma_index)]; \ + const unsigned i1 = sigma[(sigma_index) + 2 * 1]; \ + const unsigned i2 = sigma[(sigma_index) + 2 * 2]; \ + const unsigned i3 = sigma[(sigma_index) + 2 * 3]; \ + +#define SET_ROW_FROM_SIGMA_W(input) \ + SET_ROW_FROM_SIGMA_BASE(input, i0, i1, i2, i3) +#endif + + +#ifdef Z7_BLAKE2S_USE_V128_WAY2 + +#if 1 || !defined(Z7_BLAKE2S_USE_SSE41) +/* we use SET_ROW_FROM_SIGMA_BASE, that uses + (SSE4) _mm_insert_epi32(), if Z7_BLAKE2S_USE_INSERT_INSTRUCTION is defined + (SSE2) _mm_set_epi32() + MSVC can be faster for this branch: +*/ +#define AXR2_W(sigma_index, shift1, shift2) \ + { \ + AXR2_LOAD_INDEXES(sigma_index) \ + a0 = _mm_add_epi32(a0, SET_ROW_FROM_SIGMA_W(data)); \ + a1 = _mm_add_epi32(a1, SET_ROW_FROM_SIGMA_W(data + Z7_BLAKE2S_BLOCK_SIZE)); \ + AXR(a0, b0, d0, shift1) \ + AXR(a1, b1, d1, shift1) \ + AXR(c0, d0, b0, shift2) \ + AXR(c1, d1, b1, shift2) \ + } +#else +/* we use interleaved _mm_insert_epi32(): + GCC can be faster for this branch: +*/ +#define AXR2_W_PRE_INSERT(sigma_index, i) \ + { const unsigned ii = sigma[(sigma_index) + i * 2]; \ + t0 = _mm_insert_epi32(t0, *(const Int32 *)GET_SIGMA_PTR(data, ii), i); \ + t1 = _mm_insert_epi32(t1, *(const Int32 *)GET_SIGMA_PTR(data, Z7_BLAKE2S_BLOCK_SIZE + ii), i); \ + } +#define AXR2_W(sigma_index, shift1, shift2) \ + { __m128i t0, t1; \ + { const unsigned ii = sigma[sigma_index]; \ + t0 = _mm_cvtsi32_si128(*(const Int32 *)GET_SIGMA_PTR(data, ii)); \ + t1 = _mm_cvtsi32_si128(*(const Int32 *)GET_SIGMA_PTR(data, Z7_BLAKE2S_BLOCK_SIZE + ii)); \ + } \ + AXR2_W_PRE_INSERT(sigma_index, 1) \ + AXR2_W_PRE_INSERT(sigma_index, 2) \ + AXR2_W_PRE_INSERT(sigma_index, 3) \ + a0 = _mm_add_epi32(a0, t0); \ + a1 = _mm_add_epi32(a1, t1); \ + AXR(a0, b0, d0, shift1) \ + AXR(a1, b1, d1, shift1) \ + AXR(c0, d0, b0, shift2) \ + AXR(c1, d1, b1, shift2) \ + } +#endif + + +#define AXR4_W(sigma_index) \ + AXR2_W(sigma_index, 16, 12) \ + AXR2_W(sigma_index + 1, 8, 7) \ + +#define WW(r) \ + { const Byte * const sigma = k_Blake2s_Sigma_4[r]; \ + AXR4_W(0) \ + ROTATE_WORDS_TO_RIGHT(b0, 1) \ + ROTATE_WORDS_TO_RIGHT(b1, 1) \ + ROTATE_WORDS_TO_RIGHT(c0, 2) \ + ROTATE_WORDS_TO_RIGHT(c1, 2) \ + ROTATE_WORDS_TO_RIGHT(d0, 3) \ + ROTATE_WORDS_TO_RIGHT(d1, 3) \ + AXR4_W(8) \ + ROTATE_WORDS_TO_RIGHT(b0, 3) \ + ROTATE_WORDS_TO_RIGHT(b1, 3) \ + ROTATE_WORDS_TO_RIGHT(c0, 2) \ + ROTATE_WORDS_TO_RIGHT(c1, 2) \ + ROTATE_WORDS_TO_RIGHT(d0, 1) \ + ROTATE_WORDS_TO_RIGHT(d1, 1) \ + } + + +static +Z7_NO_INLINE +#ifdef BLAKE2S_ATTRIB_128BIT + BLAKE2S_ATTRIB_128BIT +#endif +void +Z7_FASTCALL +Blake2sp_Compress2_V128_Way2(UInt32 *s_items, const Byte *data, const Byte *end) +{ + size_t pos = 0; + end -= Z7_BLAKE2S_BLOCK_SIZE; + + if (data != end) + { + LOAD_ROTATE_CONSTS + do + { + UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(s_items, pos); + __m128i a0, b0, c0, d0; + __m128i a1, b1, c1, d1; + { + const __m128i inc = k_inc; + const __m128i temp = k_iv4_128; + d0 = LOAD_128_FROM_STRUCT (STATE_T(s)); + d1 = LOAD_128_FROM_STRUCT (STATE_T(s + NSW)); + D_ADD_EPI64_128(d0, inc); + D_ADD_EPI64_128(d1, inc); + STORE_128_TO_STRUCT (STATE_T(s ), d0); + STORE_128_TO_STRUCT (STATE_T(s + NSW), d1); + D_XOR_128(d0, temp); + D_XOR_128(d1, temp); + } + c1 = c0 = k_iv0_128; + a0 = LOAD_128_FROM_STRUCT(s); + b0 = LOAD_128_FROM_STRUCT(s + 4); + a1 = LOAD_128_FROM_STRUCT(s + NSW); + b1 = LOAD_128_FROM_STRUCT(s + NSW + 4); + + ROUNDS_LOOP (WW) + +#undef WW + + D_XOR_128(a0, c0); + D_XOR_128(b0, d0); + D_XOR_128(a1, c1); + D_XOR_128(b1, d1); + + D_XOR_128(a0, LOAD_128_FROM_STRUCT(s)); + D_XOR_128(b0, LOAD_128_FROM_STRUCT(s + 4)); + D_XOR_128(a1, LOAD_128_FROM_STRUCT(s + NSW)); + D_XOR_128(b1, LOAD_128_FROM_STRUCT(s + NSW + 4)); + + STORE_128_TO_STRUCT(s, a0); + STORE_128_TO_STRUCT(s + 4, b0); + STORE_128_TO_STRUCT(s + NSW, a1); + STORE_128_TO_STRUCT(s + NSW + 4, b1); + + data += Z7_BLAKE2S_BLOCK_SIZE * 2; + pos += Z7_BLAKE2S_BLOCK_SIZE * 2; + pos &= SUPER_BLOCK_MASK; + } + while (data < end); + if (data != end) + return; + } + { + UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(s_items, pos); + Z7_BLAKE2S_CompressSingleBlock(s, data); + } +} +#endif // Z7_BLAKE2S_USE_V128_WAY2 + + +#ifdef Z7_BLAKE2S_USE_V128_WAY2 + #define Z7_BLAKE2S_Compress2_V128 Blake2sp_Compress2_V128_Way2 +#else + #define Z7_BLAKE2S_Compress2_V128 Blake2sp_Compress2_V128_Way1 +#endif + + + +#ifdef Z7_BLAKE2S_MM_ROR_EPI32_IS_SUPPORTED + #define ROT_128_8(x) MM_ROR_EPI32(x, 8) + #define ROT_128_16(x) MM_ROR_EPI32(x, 16) + #define ROT_128_7(x) MM_ROR_EPI32(x, 7) + #define ROT_128_12(x) MM_ROR_EPI32(x, 12) +#else +#if defined(Z7_BLAKE2S_USE_SSSE3) || defined(Z7_BLAKE2S_USE_SSE41) + #define ROT_128_8(x) _mm_shuffle_epi8(x, r8) // k_r8 + #define ROT_128_16(x) _mm_shuffle_epi8(x, r16) // k_r16 +#else + #define ROT_128_8(x) MM_ROR_EPI32_VIA_SHIFT(x, 8) + #define ROT_128_16(x) MM_ROR_EPI32_VIA_SHIFT(x, 16) +#endif + #define ROT_128_7(x) MM_ROR_EPI32_VIA_SHIFT(x, 7) + #define ROT_128_12(x) MM_ROR_EPI32_VIA_SHIFT(x, 12) +#endif + + +#if 1 +// this branch can provide similar speed on x86* in most cases, +// because [base + index*4] provides same speed as [base + index]. +// but some compilers can generate different code with this branch, that can be faster sometimes. +// this branch uses additional table of 10*16=160 bytes. +#define SIGMA_TABLE_MULT_16( a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15) \ + SIGMA_TABLE_MULT(16, a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15) +MY_ALIGN(16) +static const Byte k_Blake2s_Sigma_16[BLAKE2S_NUM_ROUNDS][16] = + { SIGMA_TABLE(SIGMA_TABLE_MULT_16) }; +#define GET_SIGMA_PTR_128(r) const Byte * const sigma = k_Blake2s_Sigma_16[r]; +#define GET_SIGMA_VAL_128(n) (sigma[n]) +#else +#define GET_SIGMA_PTR_128(r) const Byte * const sigma = k_Blake2s_Sigma_4[r]; +#define GET_SIGMA_VAL_128(n) (4 * (size_t)sigma[n]) +#endif + + +#ifdef Z7_BLAKE2S_USE_AVX2_FAST +#if 1 +#define SIGMA_TABLE_MULT_32( a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15) \ + SIGMA_TABLE_MULT(32, a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15) +MY_ALIGN(64) +static const UInt16 k_Blake2s_Sigma_32[BLAKE2S_NUM_ROUNDS][16] = + { SIGMA_TABLE(SIGMA_TABLE_MULT_32) }; +#define GET_SIGMA_PTR_256(r) const UInt16 * const sigma = k_Blake2s_Sigma_32[r]; +#define GET_SIGMA_VAL_256(n) (sigma[n]) +#else +#define GET_SIGMA_PTR_256(r) const Byte * const sigma = k_Blake2s_Sigma_4[r]; +#define GET_SIGMA_VAL_256(n) (8 * (size_t)sigma[n]) +#endif +#endif // Z7_BLAKE2S_USE_AVX2_FAST + + +#define D_ROT_128_7(dest) dest = ROT_128_7(dest) +#define D_ROT_128_8(dest) dest = ROT_128_8(dest) +#define D_ROT_128_12(dest) dest = ROT_128_12(dest) +#define D_ROT_128_16(dest) dest = ROT_128_16(dest) + +#define OP_L(a, i) D_ADD_128 (V(a, 0), \ + LOAD_128((const Byte *)(w) + GET_SIGMA_VAL_128(2*(a)+(i)))); + +#define OP_0(a) OP_L(a, 0) +#define OP_7(a) OP_L(a, 1) + +#define OP_1(a) D_ADD_128 (V(a, 0), V(a, 1)); +#define OP_2(a) D_XOR_128 (V(a, 3), V(a, 0)); +#define OP_4(a) D_ADD_128 (V(a, 2), V(a, 3)); +#define OP_5(a) D_XOR_128 (V(a, 1), V(a, 2)); + +#define OP_3(a) D_ROT_128_16 (V(a, 3)); +#define OP_6(a) D_ROT_128_12 (V(a, 1)); +#define OP_8(a) D_ROT_128_8 (V(a, 3)); +#define OP_9(a) D_ROT_128_7 (V(a, 1)); + + +// for 32-bit x86 : interleave mode works slower, because of register pressure. + +#if 0 || 1 && (defined(MY_CPU_X86) \ + || defined(__GNUC__) && !defined(__clang__)) +// non-inteleaved version: +// is fast for x86 32-bit. +// is fast for GCC x86-64. + +#define V4G(a) \ + OP_0 (a) \ + OP_1 (a) \ + OP_2 (a) \ + OP_3 (a) \ + OP_4 (a) \ + OP_5 (a) \ + OP_6 (a) \ + OP_7 (a) \ + OP_1 (a) \ + OP_2 (a) \ + OP_8 (a) \ + OP_4 (a) \ + OP_5 (a) \ + OP_9 (a) \ + +#define V4R \ +{ \ + V4G (0) \ + V4G (1) \ + V4G (2) \ + V4G (3) \ + V4G (4) \ + V4G (5) \ + V4G (6) \ + V4G (7) \ +} + +#elif 0 || 1 && defined(MY_CPU_X86) + +#define OP_INTER_2(op, a,b) \ + op (a) \ + op (b) \ + +#define V4G(a,b) \ + OP_INTER_2 (OP_0, a,b) \ + OP_INTER_2 (OP_1, a,b) \ + OP_INTER_2 (OP_2, a,b) \ + OP_INTER_2 (OP_3, a,b) \ + OP_INTER_2 (OP_4, a,b) \ + OP_INTER_2 (OP_5, a,b) \ + OP_INTER_2 (OP_6, a,b) \ + OP_INTER_2 (OP_7, a,b) \ + OP_INTER_2 (OP_1, a,b) \ + OP_INTER_2 (OP_2, a,b) \ + OP_INTER_2 (OP_8, a,b) \ + OP_INTER_2 (OP_4, a,b) \ + OP_INTER_2 (OP_5, a,b) \ + OP_INTER_2 (OP_9, a,b) \ + +#define V4R \ +{ \ + V4G (0, 1) \ + V4G (2, 3) \ + V4G (4, 5) \ + V4G (6, 7) \ +} + +#else +// iterleave-4 version is fast for x64 (MSVC/CLANG) + +#define OP_INTER_4(op, a,b,c,d) \ + op (a) \ + op (b) \ + op (c) \ + op (d) \ + +#define V4G(a,b,c,d) \ + OP_INTER_4 (OP_0, a,b,c,d) \ + OP_INTER_4 (OP_1, a,b,c,d) \ + OP_INTER_4 (OP_2, a,b,c,d) \ + OP_INTER_4 (OP_3, a,b,c,d) \ + OP_INTER_4 (OP_4, a,b,c,d) \ + OP_INTER_4 (OP_5, a,b,c,d) \ + OP_INTER_4 (OP_6, a,b,c,d) \ + OP_INTER_4 (OP_7, a,b,c,d) \ + OP_INTER_4 (OP_1, a,b,c,d) \ + OP_INTER_4 (OP_2, a,b,c,d) \ + OP_INTER_4 (OP_8, a,b,c,d) \ + OP_INTER_4 (OP_4, a,b,c,d) \ + OP_INTER_4 (OP_5, a,b,c,d) \ + OP_INTER_4 (OP_9, a,b,c,d) \ + +#define V4R \ +{ \ + V4G (0, 1, 2, 3) \ + V4G (4, 5, 6, 7) \ +} + +#endif + +#define V4_ROUND(r) { GET_SIGMA_PTR_128(r); V4R } + + +#define V4_LOAD_MSG_1(w, m, i) \ +{ \ + __m128i m0, m1, m2, m3; \ + __m128i t0, t1, t2, t3; \ + m0 = LOADU_128((m) + ((i) + 0 * 4) * 16); \ + m1 = LOADU_128((m) + ((i) + 1 * 4) * 16); \ + m2 = LOADU_128((m) + ((i) + 2 * 4) * 16); \ + m3 = LOADU_128((m) + ((i) + 3 * 4) * 16); \ + t0 = _mm_unpacklo_epi32(m0, m1); \ + t1 = _mm_unpackhi_epi32(m0, m1); \ + t2 = _mm_unpacklo_epi32(m2, m3); \ + t3 = _mm_unpackhi_epi32(m2, m3); \ + w[(i) * 4 + 0] = _mm_unpacklo_epi64(t0, t2); \ + w[(i) * 4 + 1] = _mm_unpackhi_epi64(t0, t2); \ + w[(i) * 4 + 2] = _mm_unpacklo_epi64(t1, t3); \ + w[(i) * 4 + 3] = _mm_unpackhi_epi64(t1, t3); \ +} + +#define V4_LOAD_MSG(w, m) \ +{ \ + V4_LOAD_MSG_1 (w, m, 0) \ + V4_LOAD_MSG_1 (w, m, 1) \ + V4_LOAD_MSG_1 (w, m, 2) \ + V4_LOAD_MSG_1 (w, m, 3) \ +} + +#define V4_LOAD_UNPACK_PAIR_128(src32, i, d0, d1) \ +{ \ + const __m128i v0 = LOAD_128_FROM_STRUCT((src32) + (i ) * 4); \ + const __m128i v1 = LOAD_128_FROM_STRUCT((src32) + (i + 1) * 4); \ + d0 = _mm_unpacklo_epi32(v0, v1); \ + d1 = _mm_unpackhi_epi32(v0, v1); \ +} + +#define V4_UNPACK_PAIR_128(dest32, i, s0, s1) \ +{ \ + STORE_128_TO_STRUCT((dest32) + i * 4 , _mm_unpacklo_epi64(s0, s1)); \ + STORE_128_TO_STRUCT((dest32) + i * 4 + 16, _mm_unpackhi_epi64(s0, s1)); \ +} + +#define V4_UNPACK_STATE(dest32, src32) \ +{ \ + __m128i t0, t1, t2, t3, t4, t5, t6, t7; \ + V4_LOAD_UNPACK_PAIR_128(src32, 0, t0, t1) \ + V4_LOAD_UNPACK_PAIR_128(src32, 2, t2, t3) \ + V4_LOAD_UNPACK_PAIR_128(src32, 4, t4, t5) \ + V4_LOAD_UNPACK_PAIR_128(src32, 6, t6, t7) \ + V4_UNPACK_PAIR_128(dest32, 0, t0, t2) \ + V4_UNPACK_PAIR_128(dest32, 8, t1, t3) \ + V4_UNPACK_PAIR_128(dest32, 1, t4, t6) \ + V4_UNPACK_PAIR_128(dest32, 9, t5, t7) \ +} + + +static +Z7_NO_INLINE +#ifdef BLAKE2S_ATTRIB_128BIT + BLAKE2S_ATTRIB_128BIT +#endif +void +Z7_FASTCALL +Blake2sp_Compress2_V128_Fast(UInt32 *s_items, const Byte *data, const Byte *end) +{ + // PrintStates2(s_items, 8, 16); + size_t pos = 0; + pos /= 2; + do + { +#if defined(Z7_BLAKE2S_USE_SSSE3) && \ + !defined(Z7_BLAKE2S_MM_ROR_EPI32_IS_SUPPORTED) + const __m128i r8 = k_r8; + const __m128i r16 = k_r16; +#endif + __m128i w[16]; + __m128i v[16]; + UInt32 *s; + V4_LOAD_MSG(w, data) + s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(s_items, pos); + { + __m128i ctr = LOAD_128_FROM_STRUCT(s + 64); + D_ADD_EPI64_128 (ctr, k_inc); + STORE_128_TO_STRUCT(s + 64, ctr); + v[12] = XOR_128 (GET_128_IV_WAY4(4), _mm_shuffle_epi32(ctr, _MM_SHUFFLE(0, 0, 0, 0))); + v[13] = XOR_128 (GET_128_IV_WAY4(5), _mm_shuffle_epi32(ctr, _MM_SHUFFLE(1, 1, 1, 1))); + } + v[ 8] = GET_128_IV_WAY4(0); + v[ 9] = GET_128_IV_WAY4(1); + v[10] = GET_128_IV_WAY4(2); + v[11] = GET_128_IV_WAY4(3); + v[14] = GET_128_IV_WAY4(6); + v[15] = GET_128_IV_WAY4(7); - for (i = 0; i < 16; i++) - m[i] = GetUi32(p->buf + i * sizeof(m[i])); +#define LOAD_STATE_128_FROM_STRUCT(i) \ + v[i] = LOAD_128_FROM_STRUCT(s + (i) * 4); + +#define UPDATE_STATE_128_IN_STRUCT(i) \ + STORE_128_TO_STRUCT(s + (i) * 4, XOR_128( \ + XOR_128(v[i], v[(i) + 8]), \ + LOAD_128_FROM_STRUCT(s + (i) * 4))); - for (i = 0; i < 8; i++) - v[i] = p->h[i]; + REP8_MACRO (LOAD_STATE_128_FROM_STRUCT) + ROUNDS_LOOP (V4_ROUND) + REP8_MACRO (UPDATE_STATE_128_IN_STRUCT) + + data += Z7_BLAKE2S_BLOCK_SIZE * 4; + pos += Z7_BLAKE2S_BLOCK_SIZE * 4 / 2; + pos &= SUPER_BLOCK_SIZE / 2 - 1; } + while (data != end); +} - v[ 8] = k_Blake2s_IV[0]; - v[ 9] = k_Blake2s_IV[1]; - v[10] = k_Blake2s_IV[2]; - v[11] = k_Blake2s_IV[3]; - - v[12] = p->t[0] ^ k_Blake2s_IV[4]; - v[13] = p->t[1] ^ k_Blake2s_IV[5]; - v[14] = p->f[0] ^ k_Blake2s_IV[6]; - v[15] = p->f[1] ^ k_Blake2s_IV[7]; - #define G(r,i,a,b,c,d) \ - a += b + m[sigma[2*i+0]]; d ^= a; d = rotr32(d, 16); c += d; b ^= c; b = rotr32(b, 12); \ - a += b + m[sigma[2*i+1]]; d ^= a; d = rotr32(d, 8); c += d; b ^= c; b = rotr32(b, 7); \ +static +Z7_NO_INLINE +#ifdef BLAKE2S_ATTRIB_128BIT + BLAKE2S_ATTRIB_128BIT +#endif +void +Z7_FASTCALL +Blake2sp_Final_V128_Fast(UInt32 *states) +{ + const __m128i ctr = LOAD_128_FROM_STRUCT(states + 64); + // printf("\nBlake2sp_Compress2_V128_Fast_Final4\n"); + // PrintStates2(states, 8, 16); + { + ptrdiff_t pos = 8 * 4; + do + { + UInt32 *src32 = states + (size_t)(pos * 1); + UInt32 *dest32 = states + (size_t)(pos * 2); + V4_UNPACK_STATE(dest32, src32) + pos -= 8 * 4; + } + while (pos >= 0); + } + { + unsigned k; + for (k = 0; k < 8; k++) + { + UInt32 *s = states + (size_t)k * 16; + STORE_128_TO_STRUCT (STATE_T(s), ctr); + } + } + // PrintStates2(states, 8, 16); +} + + + +#ifdef Z7_BLAKE2S_USE_AVX2 + +#define ADD_256(a, b) _mm256_add_epi32(a, b) +#define XOR_256(a, b) _mm256_xor_si256(a, b) + +#if 1 && defined(Z7_BLAKE2S_USE_AVX512_ALWAYS) + #define MM256_ROR_EPI32 _mm256_ror_epi32 + #define Z7_MM256_ROR_EPI32_IS_SUPPORTED + #define LOAD_ROTATE_CONSTS_256 +#else +#ifdef Z7_BLAKE2S_USE_AVX2_WAY_SLOW +#ifdef Z7_BLAKE2S_USE_AVX2_WAY2 + #define LOAD_ROTATE_CONSTS_256 \ + const __m256i r8 = k_r8_256; \ + const __m256i r16 = k_r16_256; +#endif // AVX2_WAY2 + + #define MM256_ROR_EPI32(r, c) ( \ + ( 8==(c)) ? _mm256_shuffle_epi8(r,r8) \ + : (16==(c)) ? _mm256_shuffle_epi8(r,r16) \ + : _mm256_or_si256( \ + _mm256_srli_epi32((r), (c)), \ + _mm256_slli_epi32((r), 32-(c)))) +#endif // WAY_SLOW +#endif + + +#define D_ADD_256(dest, src) dest = ADD_256(dest, src) +#define D_XOR_256(dest, src) dest = XOR_256(dest, src) + +#define LOADU_256(p) _mm256_loadu_si256((const __m256i *)(const void *)(p)) + +#ifdef Z7_BLAKE2S_USE_AVX2_FAST + +#ifdef Z7_MM256_ROR_EPI32_IS_SUPPORTED +#define ROT_256_16(x) MM256_ROR_EPI32((x), 16) +#define ROT_256_12(x) MM256_ROR_EPI32((x), 12) +#define ROT_256_8(x) MM256_ROR_EPI32((x), 8) +#define ROT_256_7(x) MM256_ROR_EPI32((x), 7) +#else +#define ROTATE8 _mm256_set_epi8(12, 15, 14, 13, 8, 11, 10, 9, 4, 7, 6, 5, 0, 3, 2, 1, \ + 12, 15, 14, 13, 8, 11, 10, 9, 4, 7, 6, 5, 0, 3, 2, 1) +#define ROTATE16 _mm256_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2, \ + 13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2) +#define ROT_256_16(x) _mm256_shuffle_epi8((x), ROTATE16) +#define ROT_256_12(x) _mm256_or_si256(_mm256_srli_epi32((x), 12), _mm256_slli_epi32((x), 20)) +#define ROT_256_8(x) _mm256_shuffle_epi8((x), ROTATE8) +#define ROT_256_7(x) _mm256_or_si256(_mm256_srli_epi32((x), 7), _mm256_slli_epi32((x), 25)) +#endif + +#define D_ROT_256_7(dest) dest = ROT_256_7(dest) +#define D_ROT_256_8(dest) dest = ROT_256_8(dest) +#define D_ROT_256_12(dest) dest = ROT_256_12(dest) +#define D_ROT_256_16(dest) dest = ROT_256_16(dest) + +#define LOAD_256(p) _mm256_load_si256((const __m256i *)(const void *)(p)) +#ifdef Z7_BLAKE2SP_STRUCT_IS_NOT_ALIGNED + #define STOREU_256(p, r) _mm256_storeu_si256((__m256i *)(void *)(p), r) + #define LOAD_256_FROM_STRUCT(p) LOADU_256(p) + #define STORE_256_TO_STRUCT(p, r) STOREU_256(p, r) +#else + // if struct is aligned for 32-bytes + #define STORE_256(p, r) _mm256_store_si256((__m256i *)(void *)(p), r) + #define LOAD_256_FROM_STRUCT(p) LOAD_256(p) + #define STORE_256_TO_STRUCT(p, r) STORE_256(p, r) +#endif + +#endif // Z7_BLAKE2S_USE_AVX2_FAST + + + +#ifdef Z7_BLAKE2S_USE_AVX2_WAY_SLOW + +#if 0 + #define DIAG_PERM2(s) \ + { \ + const __m256i a = LOAD_256_FROM_STRUCT((s) ); \ + const __m256i b = LOAD_256_FROM_STRUCT((s) + NSW); \ + STORE_256_TO_STRUCT((s ), _mm256_permute2x128_si256(a, b, 0x20)); \ + STORE_256_TO_STRUCT((s + NSW), _mm256_permute2x128_si256(a, b, 0x31)); \ + } +#else + #define DIAG_PERM2(s) \ + { \ + const __m128i a = LOAD_128_FROM_STRUCT((s) + 4); \ + const __m128i b = LOAD_128_FROM_STRUCT((s) + NSW); \ + STORE_128_TO_STRUCT((s) + NSW, a); \ + STORE_128_TO_STRUCT((s) + 4 , b); \ + } +#endif + #define DIAG_PERM8(s_items) \ + { \ + DIAG_PERM2(s_items) \ + DIAG_PERM2(s_items + NSW * 2) \ + DIAG_PERM2(s_items + NSW * 4) \ + DIAG_PERM2(s_items + NSW * 6) \ + } + + +#define AXR256(a, b, d, shift) \ + D_ADD_256(a, b); \ + D_XOR_256(d, a); \ + d = MM256_ROR_EPI32(d, shift); \ + + + +#ifdef Z7_BLAKE2S_USE_GATHER + + #define TABLE_GATHER_256_4(a0,a1,a2,a3) \ + a0,a1,a2,a3, a0+16,a1+16,a2+16,a3+16 + #define TABLE_GATHER_256( \ + a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15) \ + { TABLE_GATHER_256_4(a0,a2,a4,a6), \ + TABLE_GATHER_256_4(a1,a3,a5,a7), \ + TABLE_GATHER_256_4(a8,a10,a12,a14), \ + TABLE_GATHER_256_4(a9,a11,a13,a15) } +MY_ALIGN(64) +static const UInt32 k_Blake2s_Sigma_gather256[BLAKE2S_NUM_ROUNDS][16 * 2] = + { SIGMA_TABLE(TABLE_GATHER_256) }; + #define GET_SIGMA(r) \ + const UInt32 * const sigma = k_Blake2s_Sigma_gather256[r]; + #define AXR2_LOAD_INDEXES_AVX(sigma_index) \ + const __m256i i01234567 = LOAD_256(sigma + (sigma_index)); + #define SET_ROW_FROM_SIGMA_AVX(in) \ + _mm256_i32gather_epi32((const void *)(in), i01234567, 4) + #define SIGMA_INTERLEAVE 8 + #define SIGMA_HALF_ROW_SIZE 16 + +#else // !Z7_BLAKE2S_USE_GATHER + + #define GET_SIGMA(r) \ + const Byte * const sigma = k_Blake2s_Sigma_4[r]; + #define AXR2_LOAD_INDEXES_AVX(sigma_index) \ + AXR2_LOAD_INDEXES(sigma_index) + #define SET_ROW_FROM_SIGMA_AVX(in) \ + MY_mm256_set_m128i( \ + SET_ROW_FROM_SIGMA_W((in) + Z7_BLAKE2S_BLOCK_SIZE), \ + SET_ROW_FROM_SIGMA_W(in)) + #define SIGMA_INTERLEAVE 1 + #define SIGMA_HALF_ROW_SIZE 8 +#endif // !Z7_BLAKE2S_USE_GATHER + - #define R(r) \ - G(r,0,v[ 0],v[ 4],v[ 8],v[12]) \ - G(r,1,v[ 1],v[ 5],v[ 9],v[13]) \ - G(r,2,v[ 2],v[ 6],v[10],v[14]) \ - G(r,3,v[ 3],v[ 7],v[11],v[15]) \ - G(r,4,v[ 0],v[ 5],v[10],v[15]) \ - G(r,5,v[ 1],v[ 6],v[11],v[12]) \ - G(r,6,v[ 2],v[ 7],v[ 8],v[13]) \ - G(r,7,v[ 3],v[ 4],v[ 9],v[14]) \ +#define ROTATE_WORDS_TO_RIGHT_256(a, n) \ + a = _mm256_shuffle_epi32(a, _MM_SHUFFLE((3+n)&3, (2+n)&3, (1+n)&3, (0+n)&3)); + + +#ifdef Z7_BLAKE2S_USE_AVX2_WAY2 + +#define AXR2_A(sigma_index, shift1, shift2) \ + AXR2_LOAD_INDEXES_AVX(sigma_index) \ + D_ADD_256( a0, SET_ROW_FROM_SIGMA_AVX(data)); \ + AXR256(a0, b0, d0, shift1) \ + AXR256(c0, d0, b0, shift2) \ + +#define AXR4_A(sigma_index) \ + { AXR2_A(sigma_index, 16, 12) } \ + { AXR2_A(sigma_index + SIGMA_INTERLEAVE, 8, 7) } + +#define EE1(r) \ + { GET_SIGMA(r) \ + AXR4_A(0) \ + ROTATE_WORDS_TO_RIGHT_256(b0, 1) \ + ROTATE_WORDS_TO_RIGHT_256(c0, 2) \ + ROTATE_WORDS_TO_RIGHT_256(d0, 3) \ + AXR4_A(SIGMA_HALF_ROW_SIZE) \ + ROTATE_WORDS_TO_RIGHT_256(b0, 3) \ + ROTATE_WORDS_TO_RIGHT_256(c0, 2) \ + ROTATE_WORDS_TO_RIGHT_256(d0, 1) \ + } + +static +Z7_NO_INLINE +#ifdef BLAKE2S_ATTRIB_AVX2 + BLAKE2S_ATTRIB_AVX2 +#endif +void +Z7_FASTCALL +Blake2sp_Compress2_AVX2_Way2(UInt32 *s_items, const Byte *data, const Byte *end) +{ + size_t pos = 0; + end -= Z7_BLAKE2S_BLOCK_SIZE; + + if (data != end) { - unsigned r; - for (r = 0; r < BLAKE2S_NUM_ROUNDS; r++) + LOAD_ROTATE_CONSTS_256 + DIAG_PERM8(s_items) + do { - const Byte *sigma = k_Blake2s_Sigma[r]; - R(r) + UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(s_items, pos); + __m256i a0, b0, c0, d0; + { + const __m128i inc = k_inc; + __m128i d0_128 = LOAD_128_FROM_STRUCT (STATE_T(s)); + __m128i d1_128 = LOAD_128_FROM_STRUCT (STATE_T(s + NSW)); + D_ADD_EPI64_128(d0_128, inc); + D_ADD_EPI64_128(d1_128, inc); + STORE_128_TO_STRUCT (STATE_T(s ), d0_128); + STORE_128_TO_STRUCT (STATE_T(s + NSW), d1_128); + d0 = MY_mm256_set_m128i(d1_128, d0_128); + D_XOR_256(d0, k_iv4_256); + } + c0 = SET_FROM_128(k_iv0_128); + a0 = LOAD_256_FROM_STRUCT(s + NSW * 0); + b0 = LOAD_256_FROM_STRUCT(s + NSW * 1); + + ROUNDS_LOOP (EE1) + + D_XOR_256(a0, c0); + D_XOR_256(b0, d0); + + D_XOR_256(a0, LOAD_256_FROM_STRUCT(s + NSW * 0)); + D_XOR_256(b0, LOAD_256_FROM_STRUCT(s + NSW * 1)); + + STORE_256_TO_STRUCT(s + NSW * 0, a0); + STORE_256_TO_STRUCT(s + NSW * 1, b0); + + data += Z7_BLAKE2S_BLOCK_SIZE * 2; + pos += Z7_BLAKE2S_BLOCK_SIZE * 2; + pos &= SUPER_BLOCK_MASK; } - /* R(0); R(1); R(2); R(3); R(4); R(5); R(6); R(7); R(8); R(9); */ + while (data < end); + DIAG_PERM8(s_items) + if (data != end) + return; + } + { + UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(s_items, pos); + Z7_BLAKE2S_CompressSingleBlock(s, data); } +} + +#endif // Z7_BLAKE2S_USE_AVX2_WAY2 - #undef G - #undef R + +#ifdef Z7_BLAKE2S_USE_AVX2_WAY4 + +#define AXR2_X(sigma_index, shift1, shift2) \ + AXR2_LOAD_INDEXES_AVX(sigma_index) \ + D_ADD_256( a0, SET_ROW_FROM_SIGMA_AVX(data)); \ + D_ADD_256( a1, SET_ROW_FROM_SIGMA_AVX((data) + Z7_BLAKE2S_BLOCK_SIZE * 2)); \ + AXR256(a0, b0, d0, shift1) \ + AXR256(a1, b1, d1, shift1) \ + AXR256(c0, d0, b0, shift2) \ + AXR256(c1, d1, b1, shift2) \ + +#define AXR4_X(sigma_index) \ + { AXR2_X(sigma_index, 16, 12) } \ + { AXR2_X(sigma_index + SIGMA_INTERLEAVE, 8, 7) } + +#define EE2(r) \ + { GET_SIGMA(r) \ + AXR4_X(0) \ + ROTATE_WORDS_TO_RIGHT_256(b0, 1) \ + ROTATE_WORDS_TO_RIGHT_256(b1, 1) \ + ROTATE_WORDS_TO_RIGHT_256(c0, 2) \ + ROTATE_WORDS_TO_RIGHT_256(c1, 2) \ + ROTATE_WORDS_TO_RIGHT_256(d0, 3) \ + ROTATE_WORDS_TO_RIGHT_256(d1, 3) \ + AXR4_X(SIGMA_HALF_ROW_SIZE) \ + ROTATE_WORDS_TO_RIGHT_256(b0, 3) \ + ROTATE_WORDS_TO_RIGHT_256(b1, 3) \ + ROTATE_WORDS_TO_RIGHT_256(c0, 2) \ + ROTATE_WORDS_TO_RIGHT_256(c1, 2) \ + ROTATE_WORDS_TO_RIGHT_256(d0, 1) \ + ROTATE_WORDS_TO_RIGHT_256(d1, 1) \ + } + +static +Z7_NO_INLINE +#ifdef BLAKE2S_ATTRIB_AVX2 + BLAKE2S_ATTRIB_AVX2 +#endif +void +Z7_FASTCALL +Blake2sp_Compress2_AVX2_Way4(UInt32 *s_items, const Byte *data, const Byte *end) +{ + size_t pos = 0; + + if ((size_t)(end - data) >= Z7_BLAKE2S_BLOCK_SIZE * 4) { - unsigned i; - for (i = 0; i < 8; i++) - p->h[i] ^= v[i] ^ v[i + 8]; +#ifndef Z7_MM256_ROR_EPI32_IS_SUPPORTED + const __m256i r8 = k_r8_256; + const __m256i r16 = k_r16_256; +#endif + end -= Z7_BLAKE2S_BLOCK_SIZE * 3; + DIAG_PERM8(s_items) + do + { + UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(s_items, pos); + __m256i a0, b0, c0, d0; + __m256i a1, b1, c1, d1; + { + const __m128i inc = k_inc; + __m128i d0_128 = LOAD_128_FROM_STRUCT (STATE_T(s)); + __m128i d1_128 = LOAD_128_FROM_STRUCT (STATE_T(s + NSW)); + __m128i d2_128 = LOAD_128_FROM_STRUCT (STATE_T(s + NSW * 2)); + __m128i d3_128 = LOAD_128_FROM_STRUCT (STATE_T(s + NSW * 3)); + D_ADD_EPI64_128(d0_128, inc); + D_ADD_EPI64_128(d1_128, inc); + D_ADD_EPI64_128(d2_128, inc); + D_ADD_EPI64_128(d3_128, inc); + STORE_128_TO_STRUCT (STATE_T(s ), d0_128); + STORE_128_TO_STRUCT (STATE_T(s + NSW * 1), d1_128); + STORE_128_TO_STRUCT (STATE_T(s + NSW * 2), d2_128); + STORE_128_TO_STRUCT (STATE_T(s + NSW * 3), d3_128); + d0 = MY_mm256_set_m128i(d1_128, d0_128); + d1 = MY_mm256_set_m128i(d3_128, d2_128); + D_XOR_256(d0, k_iv4_256); + D_XOR_256(d1, k_iv4_256); + } + c1 = c0 = SET_FROM_128(k_iv0_128); + a0 = LOAD_256_FROM_STRUCT(s + NSW * 0); + b0 = LOAD_256_FROM_STRUCT(s + NSW * 1); + a1 = LOAD_256_FROM_STRUCT(s + NSW * 2); + b1 = LOAD_256_FROM_STRUCT(s + NSW * 3); + + ROUNDS_LOOP (EE2) + + D_XOR_256(a0, c0); + D_XOR_256(b0, d0); + D_XOR_256(a1, c1); + D_XOR_256(b1, d1); + + D_XOR_256(a0, LOAD_256_FROM_STRUCT(s + NSW * 0)); + D_XOR_256(b0, LOAD_256_FROM_STRUCT(s + NSW * 1)); + D_XOR_256(a1, LOAD_256_FROM_STRUCT(s + NSW * 2)); + D_XOR_256(b1, LOAD_256_FROM_STRUCT(s + NSW * 3)); + + STORE_256_TO_STRUCT(s + NSW * 0, a0); + STORE_256_TO_STRUCT(s + NSW * 1, b0); + STORE_256_TO_STRUCT(s + NSW * 2, a1); + STORE_256_TO_STRUCT(s + NSW * 3, b1); + + data += Z7_BLAKE2S_BLOCK_SIZE * 4; + pos += Z7_BLAKE2S_BLOCK_SIZE * 4; + pos &= SUPER_BLOCK_MASK; + } + while (data < end); + DIAG_PERM8(s_items) + end += Z7_BLAKE2S_BLOCK_SIZE * 3; } + if (data == end) + return; + // Z7_BLAKE2S_Compress2_V128(s_items, data, end, pos); + do + { + UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(s_items, pos); + Z7_BLAKE2S_CompressSingleBlock(s, data); + data += Z7_BLAKE2S_BLOCK_SIZE; + pos += Z7_BLAKE2S_BLOCK_SIZE; + pos &= SUPER_BLOCK_MASK; + } + while (data != end); +} + +#endif // Z7_BLAKE2S_USE_AVX2_WAY4 +#endif // Z7_BLAKE2S_USE_AVX2_WAY_SLOW + + +// --------------------------------------------------------- + +#ifdef Z7_BLAKE2S_USE_AVX2_FAST + +#define OP256_L(a, i) D_ADD_256 (V(a, 0), \ + LOAD_256((const Byte *)(w) + GET_SIGMA_VAL_256(2*(a)+(i)))); + +#define OP256_0(a) OP256_L(a, 0) +#define OP256_7(a) OP256_L(a, 1) + +#define OP256_1(a) D_ADD_256 (V(a, 0), V(a, 1)); +#define OP256_2(a) D_XOR_256 (V(a, 3), V(a, 0)); +#define OP256_4(a) D_ADD_256 (V(a, 2), V(a, 3)); +#define OP256_5(a) D_XOR_256 (V(a, 1), V(a, 2)); + +#define OP256_3(a) D_ROT_256_16 (V(a, 3)); +#define OP256_6(a) D_ROT_256_12 (V(a, 1)); +#define OP256_8(a) D_ROT_256_8 (V(a, 3)); +#define OP256_9(a) D_ROT_256_7 (V(a, 1)); + + +#if 0 || 1 && defined(MY_CPU_X86) + +#define V8_G(a) \ + OP256_0 (a) \ + OP256_1 (a) \ + OP256_2 (a) \ + OP256_3 (a) \ + OP256_4 (a) \ + OP256_5 (a) \ + OP256_6 (a) \ + OP256_7 (a) \ + OP256_1 (a) \ + OP256_2 (a) \ + OP256_8 (a) \ + OP256_4 (a) \ + OP256_5 (a) \ + OP256_9 (a) \ + +#define V8R { \ + V8_G (0); \ + V8_G (1); \ + V8_G (2); \ + V8_G (3); \ + V8_G (4); \ + V8_G (5); \ + V8_G (6); \ + V8_G (7); \ +} + +#else + +#define OP256_INTER_4(op, a,b,c,d) \ + op (a) \ + op (b) \ + op (c) \ + op (d) \ + +#define V8_G(a,b,c,d) \ + OP256_INTER_4 (OP256_0, a,b,c,d) \ + OP256_INTER_4 (OP256_1, a,b,c,d) \ + OP256_INTER_4 (OP256_2, a,b,c,d) \ + OP256_INTER_4 (OP256_3, a,b,c,d) \ + OP256_INTER_4 (OP256_4, a,b,c,d) \ + OP256_INTER_4 (OP256_5, a,b,c,d) \ + OP256_INTER_4 (OP256_6, a,b,c,d) \ + OP256_INTER_4 (OP256_7, a,b,c,d) \ + OP256_INTER_4 (OP256_1, a,b,c,d) \ + OP256_INTER_4 (OP256_2, a,b,c,d) \ + OP256_INTER_4 (OP256_8, a,b,c,d) \ + OP256_INTER_4 (OP256_4, a,b,c,d) \ + OP256_INTER_4 (OP256_5, a,b,c,d) \ + OP256_INTER_4 (OP256_9, a,b,c,d) \ + +#define V8R { \ + V8_G (0, 1, 2, 3) \ + V8_G (4, 5, 6, 7) \ +} +#endif + +#define V8_ROUND(r) { GET_SIGMA_PTR_256(r); V8R } + + +// for debug: +// #define Z7_BLAKE2S_PERMUTE_WITH_GATHER +#if defined(Z7_BLAKE2S_PERMUTE_WITH_GATHER) +// gather instruction is slow. +#define V8_LOAD_MSG(w, m) \ +{ \ + unsigned i; \ + for (i = 0; i < 16; ++i) { \ + w[i] = _mm256_i32gather_epi32( \ + (const void *)((m) + i * sizeof(UInt32)),\ + _mm256_set_epi32(0x70, 0x60, 0x50, 0x40, 0x30, 0x20, 0x10, 0x00), \ + sizeof(UInt32)); \ + } \ +} +#else // !Z7_BLAKE2S_PERMUTE_WITH_GATHER + +#define V8_LOAD_MSG_2(w, a0, a1) \ +{ \ + (w)[0] = _mm256_permute2x128_si256(a0, a1, 0x20); \ + (w)[4] = _mm256_permute2x128_si256(a0, a1, 0x31); \ +} + +#define V8_LOAD_MSG_4(w, z0, z1, z2, z3) \ +{ \ + __m256i s0, s1, s2, s3; \ + s0 = _mm256_unpacklo_epi64(z0, z1); \ + s1 = _mm256_unpackhi_epi64(z0, z1); \ + s2 = _mm256_unpacklo_epi64(z2, z3); \ + s3 = _mm256_unpackhi_epi64(z2, z3); \ + V8_LOAD_MSG_2((w) + 0, s0, s2) \ + V8_LOAD_MSG_2((w) + 1, s1, s3) \ +} + +#define V8_LOAD_MSG_0(t0, t1, m) \ +{ \ + __m256i m0, m1; \ + m0 = LOADU_256(m); \ + m1 = LOADU_256((m) + 2 * 32); \ + t0 = _mm256_unpacklo_epi32(m0, m1); \ + t1 = _mm256_unpackhi_epi32(m0, m1); \ +} + +#define V8_LOAD_MSG_8(w, m) \ +{ \ + __m256i t0, t1, t2, t3, t4, t5, t6, t7; \ + V8_LOAD_MSG_0(t0, t4, (m) + 0 * 4 * 32) \ + V8_LOAD_MSG_0(t1, t5, (m) + 1 * 4 * 32) \ + V8_LOAD_MSG_0(t2, t6, (m) + 2 * 4 * 32) \ + V8_LOAD_MSG_0(t3, t7, (m) + 3 * 4 * 32) \ + V8_LOAD_MSG_4((w) , t0, t1, t2, t3) \ + V8_LOAD_MSG_4((w) + 2, t4, t5, t6, t7) \ +} + +#define V8_LOAD_MSG(w, m) \ +{ \ + V8_LOAD_MSG_8(w, m) \ + V8_LOAD_MSG_8((w) + 8, (m) + 32) \ +} + +#endif // !Z7_BLAKE2S_PERMUTE_WITH_GATHER + + +#define V8_PERM_PAIR_STORE(u, a0, a2) \ +{ \ + STORE_256_TO_STRUCT((u), _mm256_permute2x128_si256(a0, a2, 0x20)); \ + STORE_256_TO_STRUCT((u) + 8, _mm256_permute2x128_si256(a0, a2, 0x31)); \ +} + +#define V8_UNPACK_STORE_4(u, z0, z1, z2, z3) \ +{ \ + __m256i s0, s1, s2, s3; \ + s0 = _mm256_unpacklo_epi64(z0, z1); \ + s1 = _mm256_unpackhi_epi64(z0, z1); \ + s2 = _mm256_unpacklo_epi64(z2, z3); \ + s3 = _mm256_unpackhi_epi64(z2, z3); \ + V8_PERM_PAIR_STORE(u + 0, s0, s2) \ + V8_PERM_PAIR_STORE(u + 2, s1, s3) \ +} + +#define V8_UNPACK_STORE_0(src32, d0, d1) \ +{ \ + const __m256i v0 = LOAD_256_FROM_STRUCT ((src32) ); \ + const __m256i v1 = LOAD_256_FROM_STRUCT ((src32) + 8); \ + d0 = _mm256_unpacklo_epi32(v0, v1); \ + d1 = _mm256_unpackhi_epi32(v0, v1); \ +} + +#define V8_UNPACK_STATE(dest32, src32) \ +{ \ + __m256i t0, t1, t2, t3, t4, t5, t6, t7; \ + V8_UNPACK_STORE_0 ((src32) + 16 * 0, t0, t4) \ + V8_UNPACK_STORE_0 ((src32) + 16 * 1, t1, t5) \ + V8_UNPACK_STORE_0 ((src32) + 16 * 2, t2, t6) \ + V8_UNPACK_STORE_0 ((src32) + 16 * 3, t3, t7) \ + V8_UNPACK_STORE_4 ((__m256i *)(void *)(dest32) , t0, t1, t2, t3) \ + V8_UNPACK_STORE_4 ((__m256i *)(void *)(dest32) + 4, t4, t5, t6, t7) \ } -#define Blake2s_Increment_Counter(S, inc) \ - { p->t[0] += (inc); p->t[1] += (p->t[0] < (inc)); } -#define Blake2s_Set_LastBlock(p) \ - { p->f[0] = BLAKE2S_FINAL_FLAG; p->f[1] = p->lastNode_f1; } +#define V8_LOAD_STATE_256_FROM_STRUCT(i) \ + v[i] = LOAD_256_FROM_STRUCT(s_items + (i) * 8); + +#if 0 || 0 && defined(MY_CPU_X86) +#define Z7_BLAKE2S_AVX2_FAST_USE_STRUCT +#endif + +#ifdef Z7_BLAKE2S_AVX2_FAST_USE_STRUCT +// this branch doesn't use (iv) array +// so register pressure can be lower. +// it can be faster sometimes +#define V8_LOAD_STATE_256(i) V8_LOAD_STATE_256_FROM_STRUCT(i) +#define V8_UPDATE_STATE_256(i) \ +{ \ + STORE_256_TO_STRUCT(s_items + (i) * 8, XOR_256( \ + XOR_256(v[i], v[(i) + 8]), \ + LOAD_256_FROM_STRUCT(s_items + (i) * 8))); \ +} +#else +// it uses more variables (iv) registers +// it's better for gcc +// maybe that branch is better, if register pressure will be lower (avx512) +#define V8_LOAD_STATE_256(i) { iv[i] = v[i]; } +#define V8_UPDATE_STATE_256(i) { v[i] = XOR_256(XOR_256(v[i], v[i + 8]), iv[i]); } +#define V8_STORE_STATE_256(i) { STORE_256_TO_STRUCT(s_items + (i) * 8, v[i]); } +#endif -static void Blake2s_Update(CBlake2s *p, const Byte *data, size_t size) +#if 0 + // use loading constants from memory + #define KK8(n) KIV(n), KIV(n), KIV(n), KIV(n), KIV(n), KIV(n), KIV(n), KIV(n) +MY_ALIGN(64) +static const UInt32 k_Blake2s_IV_WAY8[]= { - while (size != 0) - { - unsigned pos = (unsigned)p->bufPos; - unsigned rem = BLAKE2S_BLOCK_SIZE - pos; + KK8(0), KK8(1), KK8(2), KK8(3), KK8(4), KK8(5), KK8(6), KK8(7) +}; + #define GET_256_IV_WAY8(i) LOAD_256(k_Blake2s_IV_WAY8 + 8 * (i)) +#else + // use constant generation: + #define GET_256_IV_WAY8(i) _mm256_set1_epi32((Int32)KIV(i)) +#endif - if (size <= rem) + +static +Z7_NO_INLINE +#ifdef BLAKE2S_ATTRIB_AVX2 + BLAKE2S_ATTRIB_AVX2 +#endif +void +Z7_FASTCALL +Blake2sp_Compress2_AVX2_Fast(UInt32 *s_items, const Byte *data, const Byte *end) +{ +#ifndef Z7_BLAKE2S_AVX2_FAST_USE_STRUCT + __m256i v[16]; +#endif + + // PrintStates2(s_items, 8, 16); + +#ifndef Z7_BLAKE2S_AVX2_FAST_USE_STRUCT + REP8_MACRO (V8_LOAD_STATE_256_FROM_STRUCT) +#endif + + do + { + __m256i w[16]; +#ifdef Z7_BLAKE2S_AVX2_FAST_USE_STRUCT + __m256i v[16]; +#else + __m256i iv[8]; +#endif + V8_LOAD_MSG(w, data) { - memcpy(p->buf + pos, data, size); - p->bufPos += (UInt32)size; - return; + // we use load/store ctr inside loop to reduce register pressure: +#if 1 || 1 && defined(MY_CPU_X86) + const __m256i ctr = _mm256_add_epi64( + LOAD_256_FROM_STRUCT(s_items + 64), + _mm256_set_epi32( + 0, 0, 0, Z7_BLAKE2S_BLOCK_SIZE, + 0, 0, 0, Z7_BLAKE2S_BLOCK_SIZE)); + STORE_256_TO_STRUCT(s_items + 64, ctr); +#else + const UInt64 ctr64 = *(const UInt64 *)(const void *)(s_items + 64) + + Z7_BLAKE2S_BLOCK_SIZE; + const __m256i ctr = _mm256_set_epi64x(0, (Int64)ctr64, 0, (Int64)ctr64); + *(UInt64 *)(void *)(s_items + 64) = ctr64; +#endif + v[12] = XOR_256 (GET_256_IV_WAY8(4), _mm256_shuffle_epi32(ctr, _MM_SHUFFLE(0, 0, 0, 0))); + v[13] = XOR_256 (GET_256_IV_WAY8(5), _mm256_shuffle_epi32(ctr, _MM_SHUFFLE(1, 1, 1, 1))); } + v[ 8] = GET_256_IV_WAY8(0); + v[ 9] = GET_256_IV_WAY8(1); + v[10] = GET_256_IV_WAY8(2); + v[11] = GET_256_IV_WAY8(3); + v[14] = GET_256_IV_WAY8(6); + v[15] = GET_256_IV_WAY8(7); - memcpy(p->buf + pos, data, rem); - Blake2s_Increment_Counter(S, BLAKE2S_BLOCK_SIZE) - Blake2s_Compress(p); - p->bufPos = 0; - data += rem; - size -= rem; + REP8_MACRO (V8_LOAD_STATE_256) + ROUNDS_LOOP (V8_ROUND) + REP8_MACRO (V8_UPDATE_STATE_256) + data += SUPER_BLOCK_SIZE; } + while (data != end); + +#ifndef Z7_BLAKE2S_AVX2_FAST_USE_STRUCT + REP8_MACRO (V8_STORE_STATE_256) +#endif } -static void Blake2s_Final(CBlake2s *p, Byte *digest) +static +Z7_NO_INLINE +#ifdef BLAKE2S_ATTRIB_AVX2 + BLAKE2S_ATTRIB_AVX2 +#endif +void +Z7_FASTCALL +Blake2sp_Final_AVX2_Fast(UInt32 *states) { - unsigned i; + const __m128i ctr = LOAD_128_FROM_STRUCT(states + 64); + // PrintStates2(states, 8, 16); + V8_UNPACK_STATE(states, states) + // PrintStates2(states, 8, 16); + { + unsigned k; + for (k = 0; k < 8; k++) + { + UInt32 *s = states + (size_t)k * 16; + STORE_128_TO_STRUCT (STATE_T(s), ctr); + } + } + // PrintStates2(states, 8, 16); + // printf("\nafter V8_UNPACK_STATE \n"); +} + +#endif // Z7_BLAKE2S_USE_AVX2_FAST +#endif // avx2 +#endif // vector + + +/* +#define Blake2s_Increment_Counter(s, inc) \ + { STATE_T(s)[0] += (inc); STATE_T(s)[1] += (STATE_T(s)[0] < (inc)); } +#define Blake2s_Increment_Counter_Small(s, inc) \ + { STATE_T(s)[0] += (inc); } +*/ + +#define Blake2s_Set_LastBlock(s) \ + { STATE_F(s)[0] = BLAKE2S_FINAL_FLAG; /* STATE_F(s)[1] = p->u.header.lastNode_f1; */ } + + +#if 0 || 1 && defined(Z7_MSC_VER_ORIGINAL) && Z7_MSC_VER_ORIGINAL >= 1600 + // good for vs2022 + #define LOOP_8(mac) { unsigned kkk; for (kkk = 0; kkk < 8; kkk++) mac(kkk) } +#else + // good for Z7_BLAKE2S_UNROLL for GCC9 (arm*/x86*) and MSC_VER_1400-x64. + #define LOOP_8(mac) { REP8_MACRO(mac) } +#endif + + +static +Z7_FORCE_INLINE +// Z7_NO_INLINE +void +Z7_FASTCALL +Blake2s_Compress(UInt32 *s, const Byte *input) +{ + UInt32 m[16]; + UInt32 v[16]; + { + unsigned i; + for (i = 0; i < 16; i++) + m[i] = GetUi32(input + i * 4); + } + +#define INIT_v_FROM_s(i) v[i] = s[i]; + + LOOP_8(INIT_v_FROM_s) + + // Blake2s_Increment_Counter(s, Z7_BLAKE2S_BLOCK_SIZE) + { + const UInt32 t0 = STATE_T(s)[0] + Z7_BLAKE2S_BLOCK_SIZE; + const UInt32 t1 = STATE_T(s)[1] + (t0 < Z7_BLAKE2S_BLOCK_SIZE); + STATE_T(s)[0] = t0; + STATE_T(s)[1] = t1; + v[12] = t0 ^ KIV(4); + v[13] = t1 ^ KIV(5); + } + // v[12] = STATE_T(s)[0] ^ KIV(4); + // v[13] = STATE_T(s)[1] ^ KIV(5); + v[14] = STATE_F(s)[0] ^ KIV(6); + v[15] = STATE_F(s)[1] ^ KIV(7); + + v[ 8] = KIV(0); + v[ 9] = KIV(1); + v[10] = KIV(2); + v[11] = KIV(3); + // PrintStates2((const UInt32 *)v, 1, 16); + + #define ADD_SIGMA(a, index) V(a, 0) += *(const UInt32 *)GET_SIGMA_PTR(m, sigma[index]); + #define ADD32M(dest, src, a) V(a, dest) += V(a, src); + #define XOR32M(dest, src, a) V(a, dest) ^= V(a, src); + #define RTR32M(dest, shift, a) V(a, dest) = rotrFixed(V(a, dest), shift); + +// big interleaving can provides big performance gain, if scheduler queues are small. +#if 0 || 1 && defined(MY_CPU_X86) + // interleave-1: for small register number (x86-32bit) + #define G2(index, a, x, y) \ + ADD_SIGMA (a, (index) + 2 * 0) \ + ADD32M (0, 1, a) \ + XOR32M (3, 0, a) \ + RTR32M (3, x, a) \ + ADD32M (2, 3, a) \ + XOR32M (1, 2, a) \ + RTR32M (1, y, a) \ + + #define G(a) \ + G2(a * 2 , a, 16, 12) \ + G2(a * 2 + 1, a, 8, 7) \ + + #define R2 \ + G(0) \ + G(1) \ + G(2) \ + G(3) \ + G(4) \ + G(5) \ + G(6) \ + G(7) \ + +#elif 0 || 1 && defined(MY_CPU_X86_OR_AMD64) + // interleave-2: is good if the number of registers is not big (x86-64). + + #define REP2(mac, dest, src, a, b) \ + mac(dest, src, a) \ + mac(dest, src, b) + + #define G2(index, a, b, x, y) \ + ADD_SIGMA (a, (index) + 2 * 0) \ + ADD_SIGMA (b, (index) + 2 * 1) \ + REP2 (ADD32M, 0, 1, a, b) \ + REP2 (XOR32M, 3, 0, a, b) \ + REP2 (RTR32M, 3, x, a, b) \ + REP2 (ADD32M, 2, 3, a, b) \ + REP2 (XOR32M, 1, 2, a, b) \ + REP2 (RTR32M, 1, y, a, b) \ + + #define G(a, b) \ + G2(a * 2 , a, b, 16, 12) \ + G2(a * 2 + 1, a, b, 8, 7) \ + + #define R2 \ + G(0, 1) \ + G(2, 3) \ + G(4, 5) \ + G(6, 7) \ - Blake2s_Increment_Counter(S, (UInt32)p->bufPos) - Blake2s_Set_LastBlock(p) - memset(p->buf + p->bufPos, 0, BLAKE2S_BLOCK_SIZE - p->bufPos); - Blake2s_Compress(p); +#else + // interleave-4: + // it has big register pressure for x86/x64. + // and MSVC compilers for x86/x64 are slow for this branch. + // but if we have big number of registers, this branch can be faster. - for (i = 0; i < 8; i++) + #define REP4(mac, dest, src, a, b, c, d) \ + mac(dest, src, a) \ + mac(dest, src, b) \ + mac(dest, src, c) \ + mac(dest, src, d) + + #define G2(index, a, b, c, d, x, y) \ + ADD_SIGMA (a, (index) + 2 * 0) \ + ADD_SIGMA (b, (index) + 2 * 1) \ + ADD_SIGMA (c, (index) + 2 * 2) \ + ADD_SIGMA (d, (index) + 2 * 3) \ + REP4 (ADD32M, 0, 1, a, b, c, d) \ + REP4 (XOR32M, 3, 0, a, b, c, d) \ + REP4 (RTR32M, 3, x, a, b, c, d) \ + REP4 (ADD32M, 2, 3, a, b, c, d) \ + REP4 (XOR32M, 1, 2, a, b, c, d) \ + REP4 (RTR32M, 1, y, a, b, c, d) \ + + #define G(a, b, c, d) \ + G2(a * 2 , a, b, c, d, 16, 12) \ + G2(a * 2 + 1, a, b, c, d, 8, 7) \ + + #define R2 \ + G(0, 1, 2, 3) \ + G(4, 5, 6, 7) \ + +#endif + + #define R(r) { const Byte *sigma = k_Blake2s_Sigma_4[r]; R2 } + + // Z7_BLAKE2S_UNROLL gives 5-6 KB larger code, but faster: + // 20-40% faster for (x86/x64) VC2010+/GCC/CLANG. + // 30-60% faster for (arm64-arm32) GCC. + // 5-11% faster for (arm64) CLANG-MAC. + // so Z7_BLAKE2S_UNROLL is good optimization, if there is no vector branch. + // But if there is vectors branch (for x86*), this scalar code will be unused mostly. + // So we want smaller code (without unrolling) in that case (x86*). +#if 0 || 1 && !defined(Z7_BLAKE2S_USE_VECTORS) + #define Z7_BLAKE2S_UNROLL +#endif + +#ifdef Z7_BLAKE2S_UNROLL + ROUNDS_LOOP_UNROLLED (R) +#else + ROUNDS_LOOP (R) +#endif + + #undef G + #undef G2 + #undef R + #undef R2 + + // printf("\n v after: \n"); + // PrintStates2((const UInt32 *)v, 1, 16); +#define XOR_s_PAIR_v(i) s[i] ^= v[i] ^ v[i + 8]; + + LOOP_8(XOR_s_PAIR_v) + // printf("\n s after:\n"); + // PrintStates2((const UInt32 *)s, 1, 16); +} + + +static +Z7_NO_INLINE +void +Z7_FASTCALL +Blake2sp_Compress2(UInt32 *s_items, const Byte *data, const Byte *end) +{ + size_t pos = 0; + // PrintStates2(s_items, 8, 16); + do { - SetUi32(digest + sizeof(p->h[i]) * i, p->h[i]) + UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(s_items, pos); + Blake2s_Compress(s, data); + data += Z7_BLAKE2S_BLOCK_SIZE; + pos += Z7_BLAKE2S_BLOCK_SIZE; + pos &= SUPER_BLOCK_MASK; } + while (data != end); } -/* ---------- BLAKE2s ---------- */ +#ifdef Z7_BLAKE2S_USE_VECTORS + +static Z7_BLAKE2SP_FUNC_COMPRESS g_Z7_BLAKE2SP_FUNC_COMPRESS_Fast = Blake2sp_Compress2; +static Z7_BLAKE2SP_FUNC_COMPRESS g_Z7_BLAKE2SP_FUNC_COMPRESS_Single = Blake2sp_Compress2; +static Z7_BLAKE2SP_FUNC_INIT g_Z7_BLAKE2SP_FUNC_INIT_Init; +static Z7_BLAKE2SP_FUNC_INIT g_Z7_BLAKE2SP_FUNC_INIT_Final; +static unsigned g_z7_Blake2sp_SupportedFlags; + + #define Z7_BLAKE2SP_Compress_Fast(p) (p)->u.header.func_Compress_Fast + #define Z7_BLAKE2SP_Compress_Single(p) (p)->u.header.func_Compress_Single +#else + #define Z7_BLAKE2SP_Compress_Fast(p) Blake2sp_Compress2 + #define Z7_BLAKE2SP_Compress_Single(p) Blake2sp_Compress2 +#endif // Z7_BLAKE2S_USE_VECTORS + -/* we need to xor CBlake2s::h[i] with input parameter block after Blake2s_Init0() */ +#if 1 && defined(MY_CPU_LE) + #define GET_DIGEST(_s, _digest) \ + { memcpy(_digest, _s, Z7_BLAKE2S_DIGEST_SIZE); } +#else + #define GET_DIGEST(_s, _digest) \ + { unsigned _i; for (_i = 0; _i < 8; _i++) \ + { SetUi32((_digest) + 4 * _i, (_s)[_i]) } \ + } +#endif + + +/* ---------- BLAKE2s ---------- */ /* +// we need to xor CBlake2s::h[i] with input parameter block after Blake2s_Init0() typedef struct { Byte digest_length; Byte key_length; - Byte fanout; - Byte depth; + Byte fanout; // = 1 : in sequential mode + Byte depth; // = 1 : in sequential mode UInt32 leaf_length; - Byte node_offset[6]; - Byte node_depth; - Byte inner_length; + Byte node_offset[6]; // 0 for the first, leftmost, leaf, or in sequential mode + Byte node_depth; // 0 for the leaves, or in sequential mode + Byte inner_length; // [0, 32], 0 in sequential mode Byte salt[BLAKE2S_SALTBYTES]; Byte personal[BLAKE2S_PERSONALBYTES]; } CBlake2sParam; */ +#define k_Blake2sp_IV_0 \ + (KIV(0) ^ (Z7_BLAKE2S_DIGEST_SIZE | ((UInt32)Z7_BLAKE2SP_PARALLEL_DEGREE << 16) | ((UInt32)2 << 24))) +#define k_Blake2sp_IV_3_FROM_NODE_DEPTH(node_depth) \ + (KIV(3) ^ ((UInt32)(node_depth) << 16) ^ ((UInt32)Z7_BLAKE2S_DIGEST_SIZE << 24)) -static void Blake2sp_Init_Spec(CBlake2s *p, unsigned node_offset, unsigned node_depth) +Z7_FORCE_INLINE +static void Blake2sp_Init_Spec(UInt32 *s, unsigned node_offset, unsigned node_depth) { - Blake2s_Init0(p); - - p->h[0] ^= (BLAKE2S_DIGEST_SIZE | ((UInt32)BLAKE2SP_PARALLEL_DEGREE << 16) | ((UInt32)2 << 24)); - p->h[2] ^= ((UInt32)node_offset); - p->h[3] ^= ((UInt32)node_depth << 16) | ((UInt32)BLAKE2S_DIGEST_SIZE << 24); - /* - P->digest_length = BLAKE2S_DIGEST_SIZE; - P->key_length = 0; - P->fanout = BLAKE2SP_PARALLEL_DEGREE; - P->depth = 2; - P->leaf_length = 0; - store48(P->node_offset, node_offset); - P->node_depth = node_depth; - P->inner_length = BLAKE2S_DIGEST_SIZE; - */ + s[0] = k_Blake2sp_IV_0; + s[1] = KIV(1); + s[2] = KIV(2) ^ (UInt32)node_offset; + s[3] = k_Blake2sp_IV_3_FROM_NODE_DEPTH(node_depth); + s[4] = KIV(4); + s[5] = KIV(5); + s[6] = KIV(6); + s[7] = KIV(7); + + STATE_T(s)[0] = 0; + STATE_T(s)[1] = 0; + STATE_F(s)[0] = 0; + STATE_F(s)[1] = 0; } +#ifdef Z7_BLAKE2S_USE_V128_FAST + +static +Z7_NO_INLINE +#ifdef BLAKE2S_ATTRIB_128BIT + BLAKE2S_ATTRIB_128BIT +#endif +void +Z7_FASTCALL +Blake2sp_InitState_V128_Fast(UInt32 *states) +{ +#define STORE_128_PAIR_INIT_STATES_2(i, t0, t1) \ + { STORE_128_TO_STRUCT(states + 0 + 4 * (i), (t0)); \ + STORE_128_TO_STRUCT(states + 32 + 4 * (i), (t1)); \ + } +#define STORE_128_PAIR_INIT_STATES_1(i, mac) \ + { const __m128i t = mac; \ + STORE_128_PAIR_INIT_STATES_2(i, t, t) \ + } +#define STORE_128_PAIR_INIT_STATES_IV(i) \ + STORE_128_PAIR_INIT_STATES_1(i, GET_128_IV_WAY4(i)) + + STORE_128_PAIR_INIT_STATES_1 (0, _mm_set1_epi32((Int32)k_Blake2sp_IV_0)) + STORE_128_PAIR_INIT_STATES_IV (1) + { + const __m128i t = GET_128_IV_WAY4(2); + STORE_128_PAIR_INIT_STATES_2 (2, + XOR_128(t, _mm_set_epi32(3, 2, 1, 0)), + XOR_128(t, _mm_set_epi32(7, 6, 5, 4))) + } + STORE_128_PAIR_INIT_STATES_1 (3, _mm_set1_epi32((Int32)k_Blake2sp_IV_3_FROM_NODE_DEPTH(0))) + STORE_128_PAIR_INIT_STATES_IV (4) + STORE_128_PAIR_INIT_STATES_IV (5) + STORE_128_PAIR_INIT_STATES_IV (6) + STORE_128_PAIR_INIT_STATES_IV (7) + STORE_128_PAIR_INIT_STATES_1 (16, _mm_set_epi32(0, 0, 0, 0)) + // printf("\n== exit Blake2sp_InitState_V128_Fast ctr=%d\n", states[64]); +} + +#endif // Z7_BLAKE2S_USE_V128_FAST + + +#ifdef Z7_BLAKE2S_USE_AVX2_FAST + +static +Z7_NO_INLINE +#ifdef BLAKE2S_ATTRIB_AVX2 + BLAKE2S_ATTRIB_AVX2 +#endif +void +Z7_FASTCALL +Blake2sp_InitState_AVX2_Fast(UInt32 *states) +{ +#define STORE_256_INIT_STATES(i, t) \ + STORE_256_TO_STRUCT(states + 8 * (i), t); +#define STORE_256_INIT_STATES_IV(i) \ + STORE_256_INIT_STATES(i, GET_256_IV_WAY8(i)) + + STORE_256_INIT_STATES (0, _mm256_set1_epi32((Int32)k_Blake2sp_IV_0)) + STORE_256_INIT_STATES_IV (1) + STORE_256_INIT_STATES (2, XOR_256( GET_256_IV_WAY8(2), + _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0))) + STORE_256_INIT_STATES (3, _mm256_set1_epi32((Int32)k_Blake2sp_IV_3_FROM_NODE_DEPTH(0))) + STORE_256_INIT_STATES_IV (4) + STORE_256_INIT_STATES_IV (5) + STORE_256_INIT_STATES_IV (6) + STORE_256_INIT_STATES_IV (7) + STORE_256_INIT_STATES (8, _mm256_set_epi32(0, 0, 0, 0, 0, 0, 0, 0)) + // printf("\n== exit Blake2sp_InitState_AVX2_Fast\n"); +} + +#endif // Z7_BLAKE2S_USE_AVX2_FAST + + + +Z7_NO_INLINE +void Blake2sp_InitState(CBlake2sp *p) +{ + size_t i; + // memset(p->states, 0, sizeof(p->states)); // for debug + p->u.header.cycPos = 0; +#ifdef Z7_BLAKE2SP_USE_FUNCTIONS + if (p->u.header.func_Init) + { + p->u.header.func_Init(p->states); + return; + } +#endif + for (i = 0; i < Z7_BLAKE2SP_PARALLEL_DEGREE; i++) + Blake2sp_Init_Spec(p->states + i * NSW, (unsigned)i, 0); +} + void Blake2sp_Init(CBlake2sp *p) { - unsigned i; - - p->bufPos = 0; +#ifdef Z7_BLAKE2SP_USE_FUNCTIONS + p->u.header.func_Compress_Fast = +#ifdef Z7_BLAKE2S_USE_VECTORS + g_Z7_BLAKE2SP_FUNC_COMPRESS_Fast; +#else + NULL; +#endif + + p->u.header.func_Compress_Single = +#ifdef Z7_BLAKE2S_USE_VECTORS + g_Z7_BLAKE2SP_FUNC_COMPRESS_Single; +#else + NULL; +#endif + + p->u.header.func_Init = +#ifdef Z7_BLAKE2S_USE_VECTORS + g_Z7_BLAKE2SP_FUNC_INIT_Init; +#else + NULL; +#endif - for (i = 0; i < BLAKE2SP_PARALLEL_DEGREE; i++) - Blake2sp_Init_Spec(&p->S[i], i, 0); + p->u.header.func_Final = +#ifdef Z7_BLAKE2S_USE_VECTORS + g_Z7_BLAKE2SP_FUNC_INIT_Final; +#else + NULL; +#endif +#endif - p->S[BLAKE2SP_PARALLEL_DEGREE - 1].lastNode_f1 = BLAKE2S_FINAL_FLAG; + Blake2sp_InitState(p); } void Blake2sp_Update(CBlake2sp *p, const Byte *data, size_t size) { - unsigned pos = p->bufPos; - while (size != 0) + size_t pos; + // printf("\nsize = 0x%6x, cycPos = %5u data = %p\n", (unsigned)size, (unsigned)p->u.header.cycPos, data); + if (size == 0) + return; + pos = p->u.header.cycPos; + // pos < SUPER_BLOCK_SIZE * 2 : is expected + // pos == SUPER_BLOCK_SIZE * 2 : is not expected, but is supported also + { + const size_t pos2 = pos & SUPER_BLOCK_MASK; + if (pos2) + { + const size_t rem = SUPER_BLOCK_SIZE - pos2; + if (rem > size) + { + p->u.header.cycPos = (unsigned)(pos + size); + // cycPos < SUPER_BLOCK_SIZE * 2 + memcpy((Byte *)(void *)p->buf32 + pos, data, size); + /* to simpilify the code here we don't try to process first superblock, + if (cycPos > SUPER_BLOCK_SIZE * 2 - Z7_BLAKE2S_BLOCK_SIZE) */ + return; + } + // (rem <= size) + memcpy((Byte *)(void *)p->buf32 + pos, data, rem); + pos += rem; + data += rem; + size -= rem; + } + } + + // pos <= SUPER_BLOCK_SIZE * 2 + // pos % SUPER_BLOCK_SIZE == 0 + if (pos) + { + /* pos == SUPER_BLOCK_SIZE || + pos == SUPER_BLOCK_SIZE * 2 */ + size_t end = pos; + if (size > SUPER_BLOCK_SIZE - Z7_BLAKE2S_BLOCK_SIZE + || (end -= SUPER_BLOCK_SIZE)) + { + Z7_BLAKE2SP_Compress_Fast(p)(p->states, + (const Byte *)(const void *)p->buf32, + (const Byte *)(const void *)p->buf32 + end); + if (pos -= end) + memcpy(p->buf32, (const Byte *)(const void *)p->buf32 + + SUPER_BLOCK_SIZE, SUPER_BLOCK_SIZE); + } + } + + // pos == 0 || (pos == SUPER_BLOCK_SIZE && size <= SUPER_BLOCK_SIZE - Z7_BLAKE2S_BLOCK_SIZE) + if (size > SUPER_BLOCK_SIZE * 2 - Z7_BLAKE2S_BLOCK_SIZE) + { + // pos == 0 + const Byte *end; + const size_t size2 = (size - (SUPER_BLOCK_SIZE - Z7_BLAKE2S_BLOCK_SIZE + 1)) + & ~(size_t)SUPER_BLOCK_MASK; + size -= size2; + // size < SUPER_BLOCK_SIZE * 2 + end = data + size2; + Z7_BLAKE2SP_Compress_Fast(p)(p->states, data, end); + data = end; + } + + if (size != 0) { - unsigned index = pos / BLAKE2S_BLOCK_SIZE; - unsigned rem = BLAKE2S_BLOCK_SIZE - (pos & (BLAKE2S_BLOCK_SIZE - 1)); - if (rem > size) - rem = (unsigned)size; - Blake2s_Update(&p->S[index], data, rem); - size -= rem; - data += rem; - pos += rem; - pos &= (BLAKE2S_BLOCK_SIZE * BLAKE2SP_PARALLEL_DEGREE - 1); + memcpy((Byte *)(void *)p->buf32 + pos, data, size); + pos += size; } - p->bufPos = pos; + p->u.header.cycPos = (unsigned)pos; + // cycPos < SUPER_BLOCK_SIZE * 2 } void Blake2sp_Final(CBlake2sp *p, Byte *digest) { - CBlake2s R; - unsigned i; + // UInt32 * const R_states = p->states; + // printf("\nBlake2sp_Final \n"); +#ifdef Z7_BLAKE2SP_USE_FUNCTIONS + if (p->u.header.func_Final) + p->u.header.func_Final(p->states); +#endif + // printf("\n=====\nBlake2sp_Final \n"); + // PrintStates(p->states, 32); + + // (p->u.header.cycPos == SUPER_BLOCK_SIZE) can be processed in any branch: + if (p->u.header.cycPos <= SUPER_BLOCK_SIZE) + { + unsigned pos; + memset((Byte *)(void *)p->buf32 + p->u.header.cycPos, + 0, SUPER_BLOCK_SIZE - p->u.header.cycPos); + STATE_F(&p->states[(Z7_BLAKE2SP_PARALLEL_DEGREE - 1) * NSW])[1] = BLAKE2S_FINAL_FLAG; + for (pos = 0; pos < SUPER_BLOCK_SIZE; pos += Z7_BLAKE2S_BLOCK_SIZE) + { + UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(p->states, pos); + Blake2s_Set_LastBlock(s) + if (pos + Z7_BLAKE2S_BLOCK_SIZE > p->u.header.cycPos) + { + UInt32 delta = Z7_BLAKE2S_BLOCK_SIZE; + if (pos < p->u.header.cycPos) + delta -= p->u.header.cycPos & (Z7_BLAKE2S_BLOCK_SIZE - 1); + // 0 < delta <= Z7_BLAKE2S_BLOCK_SIZE + { + const UInt32 v = STATE_T(s)[0]; + STATE_T(s)[1] -= v < delta; // (v < delta) is same condition here as (v == 0) + STATE_T(s)[0] = v - delta; + } + } + } + // PrintStates(p->states, 16); + Z7_BLAKE2SP_Compress_Single(p)(p->states, + (Byte *)(void *)p->buf32, + (Byte *)(void *)p->buf32 + SUPER_BLOCK_SIZE); + // PrintStates(p->states, 16); + } + else + { + // (p->u.header.cycPos > SUPER_BLOCK_SIZE) + unsigned pos; + for (pos = 0; pos < SUPER_BLOCK_SIZE; pos += Z7_BLAKE2S_BLOCK_SIZE) + { + UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(p->states, pos); + if (pos + SUPER_BLOCK_SIZE >= p->u.header.cycPos) + Blake2s_Set_LastBlock(s) + } + if (p->u.header.cycPos <= SUPER_BLOCK_SIZE * 2 - Z7_BLAKE2S_BLOCK_SIZE) + STATE_F(&p->states[(Z7_BLAKE2SP_PARALLEL_DEGREE - 1) * NSW])[1] = BLAKE2S_FINAL_FLAG; + + Z7_BLAKE2SP_Compress_Single(p)(p->states, + (Byte *)(void *)p->buf32, + (Byte *)(void *)p->buf32 + SUPER_BLOCK_SIZE); - Blake2sp_Init_Spec(&R, 0, 1); - R.lastNode_f1 = BLAKE2S_FINAL_FLAG; + // if (p->u.header.cycPos > SUPER_BLOCK_SIZE * 2 - Z7_BLAKE2S_BLOCK_SIZE; + STATE_F(&p->states[(Z7_BLAKE2SP_PARALLEL_DEGREE - 1) * NSW])[1] = BLAKE2S_FINAL_FLAG; + + // if (p->u.header.cycPos != SUPER_BLOCK_SIZE) + { + pos = SUPER_BLOCK_SIZE; + for (;;) + { + UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(p->states, pos & SUPER_BLOCK_MASK); + Blake2s_Set_LastBlock(s) + pos += Z7_BLAKE2S_BLOCK_SIZE; + if (pos >= p->u.header.cycPos) + { + if (pos != p->u.header.cycPos) + { + const UInt32 delta = pos - p->u.header.cycPos; + const UInt32 v = STATE_T(s)[0]; + STATE_T(s)[1] -= v < delta; + STATE_T(s)[0] = v - delta; + memset((Byte *)(void *)p->buf32 + p->u.header.cycPos, 0, delta); + } + break; + } + } + Z7_BLAKE2SP_Compress_Single(p)(p->states, + (Byte *)(void *)p->buf32 + SUPER_BLOCK_SIZE, + (Byte *)(void *)p->buf32 + pos); + } + } - for (i = 0; i < BLAKE2SP_PARALLEL_DEGREE; i++) { - Byte hash[BLAKE2S_DIGEST_SIZE]; - Blake2s_Final(&p->S[i], hash); - Blake2s_Update(&R, hash, BLAKE2S_DIGEST_SIZE); + size_t pos; + for (pos = 0; pos < SUPER_BLOCK_SIZE / 2; pos += Z7_BLAKE2S_BLOCK_SIZE / 2) + { + const UInt32 * const s = GET_STATE_TABLE_PTR_FROM_BYTE_POS(p->states, (pos * 2)); + Byte *dest = (Byte *)(void *)p->buf32 + pos; + GET_DIGEST(s, dest) + } } + Blake2sp_Init_Spec(p->states, 0, 1); + { + size_t pos; + for (pos = 0; pos < (Z7_BLAKE2SP_PARALLEL_DEGREE * Z7_BLAKE2S_DIGEST_SIZE) + - Z7_BLAKE2S_BLOCK_SIZE; pos += Z7_BLAKE2S_BLOCK_SIZE) + { + Z7_BLAKE2SP_Compress_Single(p)(p->states, + (const Byte *)(const void *)p->buf32 + pos, + (const Byte *)(const void *)p->buf32 + pos + Z7_BLAKE2S_BLOCK_SIZE); + } + } + // Blake2s_Final(p->states, 0, digest, p, (Byte *)(void *)p->buf32 + i); + Blake2s_Set_LastBlock(p->states) + STATE_F(p->states)[1] = BLAKE2S_FINAL_FLAG; + { + Z7_BLAKE2SP_Compress_Single(p)(p->states, + (const Byte *)(const void *)p->buf32 + Z7_BLAKE2SP_PARALLEL_DEGREE / 2 * Z7_BLAKE2S_BLOCK_SIZE - Z7_BLAKE2S_BLOCK_SIZE, + (const Byte *)(const void *)p->buf32 + Z7_BLAKE2SP_PARALLEL_DEGREE / 2 * Z7_BLAKE2S_BLOCK_SIZE); + } + GET_DIGEST(p->states, digest) + // printf("\n Blake2sp_Final 555 numDataInBufs = %5u\n", (unsigned)p->u.header.numDataInBufs); +} + - Blake2s_Final(&R, digest); +BoolInt Blake2sp_SetFunction(CBlake2sp *p, unsigned algo) +{ + // printf("\n========== setfunction = %d ======== \n", algo); +#ifdef Z7_BLAKE2SP_USE_FUNCTIONS + Z7_BLAKE2SP_FUNC_COMPRESS func = NULL; + Z7_BLAKE2SP_FUNC_COMPRESS func_Single = NULL; + Z7_BLAKE2SP_FUNC_INIT func_Final = NULL; + Z7_BLAKE2SP_FUNC_INIT func_Init = NULL; +#else + UNUSED_VAR(p) +#endif + +#ifdef Z7_BLAKE2S_USE_VECTORS + + func = func_Single = Blake2sp_Compress2; + + if (algo != Z7_BLAKE2SP_ALGO_SCALAR) + { + // printf("\n========== setfunction NON-SCALER ======== \n"); + if (algo == Z7_BLAKE2SP_ALGO_DEFAULT) + { + func = g_Z7_BLAKE2SP_FUNC_COMPRESS_Fast; + func_Single = g_Z7_BLAKE2SP_FUNC_COMPRESS_Single; + func_Init = g_Z7_BLAKE2SP_FUNC_INIT_Init; + func_Final = g_Z7_BLAKE2SP_FUNC_INIT_Final; + } + else + { + if ((g_z7_Blake2sp_SupportedFlags & (1u << algo)) == 0) + return False; + +#ifdef Z7_BLAKE2S_USE_AVX2 + + func_Single = +#if defined(Z7_BLAKE2S_USE_AVX2_WAY2) + Blake2sp_Compress2_AVX2_Way2; +#else + Z7_BLAKE2S_Compress2_V128; +#endif + +#ifdef Z7_BLAKE2S_USE_AVX2_FAST + if (algo == Z7_BLAKE2SP_ALGO_V256_FAST) + { + func = Blake2sp_Compress2_AVX2_Fast; + func_Final = Blake2sp_Final_AVX2_Fast; + func_Init = Blake2sp_InitState_AVX2_Fast; + } + else +#endif +#ifdef Z7_BLAKE2S_USE_AVX2_WAY2 + if (algo == Z7_BLAKE2SP_ALGO_V256_WAY2) + func = Blake2sp_Compress2_AVX2_Way2; + else +#endif +#ifdef Z7_BLAKE2S_USE_AVX2_WAY4 + if (algo == Z7_BLAKE2SP_ALGO_V256_WAY4) + { + func_Single = func = Blake2sp_Compress2_AVX2_Way4; + } + else +#endif +#endif // avx2 + { + if (algo == Z7_BLAKE2SP_ALGO_V128_FAST) + { + func = Blake2sp_Compress2_V128_Fast; + func_Final = Blake2sp_Final_V128_Fast; + func_Init = Blake2sp_InitState_V128_Fast; + func_Single = Z7_BLAKE2S_Compress2_V128; + } + else +#ifdef Z7_BLAKE2S_USE_V128_WAY2 + if (algo == Z7_BLAKE2SP_ALGO_V128_WAY2) + func = func_Single = Blake2sp_Compress2_V128_Way2; + else +#endif + { + if (algo != Z7_BLAKE2SP_ALGO_V128_WAY1) + return False; + func = func_Single = Blake2sp_Compress2_V128_Way1; + } + } + } + } +#else // !VECTORS + if (algo > 1) // Z7_BLAKE2SP_ALGO_SCALAR + return False; +#endif // !VECTORS + +#ifdef Z7_BLAKE2SP_USE_FUNCTIONS + p->u.header.func_Compress_Fast = func; + p->u.header.func_Compress_Single = func_Single; + p->u.header.func_Final = func_Final; + p->u.header.func_Init = func_Init; +#endif + // printf("\n p->u.header.func_Compress = %p", p->u.header.func_Compress); + return True; +} + + +void z7_Black2sp_Prepare(void) +{ +#ifdef Z7_BLAKE2S_USE_VECTORS + unsigned flags = 0; // (1u << Z7_BLAKE2SP_ALGO_V128_SCALAR); + + Z7_BLAKE2SP_FUNC_COMPRESS func_Fast = Blake2sp_Compress2; + Z7_BLAKE2SP_FUNC_COMPRESS func_Single = Blake2sp_Compress2; + Z7_BLAKE2SP_FUNC_INIT func_Init = NULL; + Z7_BLAKE2SP_FUNC_INIT func_Final = NULL; + +#if defined(MY_CPU_X86_OR_AMD64) + #if defined(Z7_BLAKE2S_USE_AVX512_ALWAYS) + if (CPU_IsSupported_AVX512F_AVX512VL()) + #endif + #if defined(Z7_BLAKE2S_USE_SSE41) + if (CPU_IsSupported_SSE41()) + #elif defined(Z7_BLAKE2S_USE_SSSE3) + if (CPU_IsSupported_SSSE3()) + #elif !defined(MY_CPU_AMD64) + if (CPU_IsSupported_SSE2()) + #endif +#endif + { + #if defined(Z7_BLAKE2S_USE_SSE41) + // printf("\n========== Blake2s SSE41 128-bit\n"); + #elif defined(Z7_BLAKE2S_USE_SSSE3) + // printf("\n========== Blake2s SSSE3 128-bit\n"); + #else + // printf("\n========== Blake2s SSE2 128-bit\n"); + #endif + // func_Fast = f_vector = Blake2sp_Compress2_V128_Way2; + // printf("\n========== Blake2sp_Compress2_V128_Way2\n"); + func_Fast = + func_Single = Z7_BLAKE2S_Compress2_V128; + flags |= (1u << Z7_BLAKE2SP_ALGO_V128_WAY1); +#ifdef Z7_BLAKE2S_USE_V128_WAY2 + flags |= (1u << Z7_BLAKE2SP_ALGO_V128_WAY2); +#endif +#ifdef Z7_BLAKE2S_USE_V128_FAST + flags |= (1u << Z7_BLAKE2SP_ALGO_V128_FAST); + func_Fast = Blake2sp_Compress2_V128_Fast; + func_Init = Blake2sp_InitState_V128_Fast; + func_Final = Blake2sp_Final_V128_Fast; +#endif + +#ifdef Z7_BLAKE2S_USE_AVX2 +#if defined(MY_CPU_X86_OR_AMD64) + if ( + #if 0 && defined(Z7_BLAKE2S_USE_AVX512_ALWAYS) + CPU_IsSupported_AVX512F_AVX512VL() && + #endif + CPU_IsSupported_AVX2() + ) +#endif + { + // #pragma message ("=== Blake2s AVX2") + // printf("\n========== Blake2s AVX2\n"); + +#ifdef Z7_BLAKE2S_USE_AVX2_WAY2 + func_Single = Blake2sp_Compress2_AVX2_Way2; + flags |= (1u << Z7_BLAKE2SP_ALGO_V256_WAY2); +#endif +#ifdef Z7_BLAKE2S_USE_AVX2_WAY4 + flags |= (1u << Z7_BLAKE2SP_ALGO_V256_WAY4); +#endif + +#ifdef Z7_BLAKE2S_USE_AVX2_FAST + flags |= (1u << Z7_BLAKE2SP_ALGO_V256_FAST); + func_Fast = Blake2sp_Compress2_AVX2_Fast; + func_Init = Blake2sp_InitState_AVX2_Fast; + func_Final = Blake2sp_Final_AVX2_Fast; +#elif defined(Z7_BLAKE2S_USE_AVX2_WAY4) + func_Fast = Blake2sp_Compress2_AVX2_Way4; +#elif defined(Z7_BLAKE2S_USE_AVX2_WAY2) + func_Fast = Blake2sp_Compress2_AVX2_Way2; +#endif + } // avx2 +#endif // avx2 + } // sse* + g_Z7_BLAKE2SP_FUNC_COMPRESS_Fast = func_Fast; + g_Z7_BLAKE2SP_FUNC_COMPRESS_Single = func_Single; + g_Z7_BLAKE2SP_FUNC_INIT_Init = func_Init; + g_Z7_BLAKE2SP_FUNC_INIT_Final = func_Final; + g_z7_Blake2sp_SupportedFlags = flags; + // printf("\nflags=%x\n", flags); +#endif // vectors } -#undef rotr32 +/* +#ifdef Z7_BLAKE2S_USE_VECTORS +void align_test2(CBlake2sp *sp); +void align_test2(CBlake2sp *sp) +{ + __m128i a = LOAD_128(sp->states); + D_XOR_128(a, LOAD_128(sp->states + 4)); + STORE_128(sp->states, a); +} +void align_test2(void); +void align_test2(void) +{ + CBlake2sp sp; + Blake2sp_Init(&sp); + Blake2sp_Update(&sp, NULL, 0); +} +#endif +*/ diff --git a/C/Bra.c b/C/Bra.c index 22e0e47..e61edf8 100644 --- a/C/Bra.c +++ b/C/Bra.c @@ -1,11 +1,11 @@ /* Bra.c -- Branch converters for RISC code -2023-04-02 : Igor Pavlov : Public domain */ +2024-01-20 : Igor Pavlov : Public domain */ #include "Precomp.h" #include "Bra.h" -#include "CpuArch.h" #include "RotateDefs.h" +#include "CpuArch.h" #if defined(MY_CPU_SIZEOF_POINTER) \ && ( MY_CPU_SIZEOF_POINTER == 4 \ @@ -26,7 +26,7 @@ #define BR_CONVERT_VAL(v, c) if (encoding) v += c; else v -= c; // #define BR_CONVERT_VAL(v, c) if (!encoding) c = (UInt32)0 - c; v += c; -#define Z7_BRANCH_CONV(name) z7_BranchConv_ ## name +#define Z7_BRANCH_CONV(name) z7_ ## name #define Z7_BRANCH_FUNC_MAIN(name) \ static \ @@ -42,11 +42,11 @@ Byte *m(name)(Byte *data, SizeT size, UInt32 pc) \ #ifdef Z7_EXTRACT_ONLY #define Z7_BRANCH_FUNCS_IMP(name) \ - Z7_BRANCH_FUNC_IMP(name, Z7_BRANCH_CONV_DEC, 0) + Z7_BRANCH_FUNC_IMP(name, Z7_BRANCH_CONV_DEC_2, 0) #else #define Z7_BRANCH_FUNCS_IMP(name) \ - Z7_BRANCH_FUNC_IMP(name, Z7_BRANCH_CONV_DEC, 0) \ - Z7_BRANCH_FUNC_IMP(name, Z7_BRANCH_CONV_ENC, 1) + Z7_BRANCH_FUNC_IMP(name, Z7_BRANCH_CONV_DEC_2, 0) \ + Z7_BRANCH_FUNC_IMP(name, Z7_BRANCH_CONV_ENC_2, 1) #endif #if defined(__clang__) @@ -72,7 +72,7 @@ Byte *m(name)(Byte *data, SizeT size, UInt32 pc) \ #endif -Z7_BRANCH_FUNC_MAIN(ARM64) +Z7_BRANCH_FUNC_MAIN(BranchConv_ARM64) { // Byte *p = data; const Byte *lim; @@ -121,10 +121,10 @@ Z7_BRANCH_FUNC_MAIN(ARM64) } } } -Z7_BRANCH_FUNCS_IMP(ARM64) +Z7_BRANCH_FUNCS_IMP(BranchConv_ARM64) -Z7_BRANCH_FUNC_MAIN(ARM) +Z7_BRANCH_FUNC_MAIN(BranchConv_ARM) { // Byte *p = data; const Byte *lim; @@ -152,10 +152,10 @@ Z7_BRANCH_FUNC_MAIN(ARM) } } } -Z7_BRANCH_FUNCS_IMP(ARM) +Z7_BRANCH_FUNCS_IMP(BranchConv_ARM) -Z7_BRANCH_FUNC_MAIN(PPC) +Z7_BRANCH_FUNC_MAIN(BranchConv_PPC) { // Byte *p = data; const Byte *lim; @@ -192,14 +192,14 @@ Z7_BRANCH_FUNC_MAIN(PPC) } } } -Z7_BRANCH_FUNCS_IMP(PPC) +Z7_BRANCH_FUNCS_IMP(BranchConv_PPC) #ifdef Z7_CPU_FAST_ROTATE_SUPPORTED #define BR_SPARC_USE_ROTATE #endif -Z7_BRANCH_FUNC_MAIN(SPARC) +Z7_BRANCH_FUNC_MAIN(BranchConv_SPARC) { // Byte *p = data; const Byte *lim; @@ -254,10 +254,10 @@ Z7_BRANCH_FUNC_MAIN(SPARC) } } } -Z7_BRANCH_FUNCS_IMP(SPARC) +Z7_BRANCH_FUNCS_IMP(BranchConv_SPARC) -Z7_BRANCH_FUNC_MAIN(ARMT) +Z7_BRANCH_FUNC_MAIN(BranchConv_ARMT) { // Byte *p = data; Byte *lim; @@ -335,12 +335,12 @@ Z7_BRANCH_FUNC_MAIN(ARMT) // return (Byte *)(lim + (((lim[1] ^ ~0xfu) & ~7u) == 0 ? 0 : 2)); // return (Byte *)(lim + 2 - (((((unsigned)lim[1] ^ 8) + 8) >> 7) & 2)); } -Z7_BRANCH_FUNCS_IMP(ARMT) +Z7_BRANCH_FUNCS_IMP(BranchConv_ARMT) // #define BR_IA64_NO_INLINE -Z7_BRANCH_FUNC_MAIN(IA64) +Z7_BRANCH_FUNC_MAIN(BranchConv_IA64) { // Byte *p = data; const Byte *lim; @@ -417,4 +417,293 @@ Z7_BRANCH_FUNC_MAIN(IA64) } } } -Z7_BRANCH_FUNCS_IMP(IA64) +Z7_BRANCH_FUNCS_IMP(BranchConv_IA64) + + +#define BR_CONVERT_VAL_ENC(v) v += BR_PC_GET; +#define BR_CONVERT_VAL_DEC(v) v -= BR_PC_GET; + +#if 1 && defined(MY_CPU_LE_UNALIGN) + #define RISCV_USE_UNALIGNED_LOAD +#endif + +#ifdef RISCV_USE_UNALIGNED_LOAD + #define RISCV_GET_UI32(p) GetUi32(p) + #define RISCV_SET_UI32(p, v) { SetUi32(p, v) } +#else + #define RISCV_GET_UI32(p) \ + ((UInt32)GetUi16a(p) + \ + ((UInt32)GetUi16a((p) + 2) << 16)) + #define RISCV_SET_UI32(p, v) { \ + SetUi16a(p, (UInt16)(v)) \ + SetUi16a((p) + 2, (UInt16)(v >> 16)) } +#endif + +#if 1 && defined(MY_CPU_LE) + #define RISCV_USE_16BIT_LOAD +#endif + +#ifdef RISCV_USE_16BIT_LOAD + #define RISCV_LOAD_VAL(p) GetUi16a(p) +#else + #define RISCV_LOAD_VAL(p) (*(p)) +#endif + +#define RISCV_INSTR_SIZE 2 +#define RISCV_STEP_1 (4 + RISCV_INSTR_SIZE) +#define RISCV_STEP_2 4 +#define RISCV_REG_VAL (2 << 7) +#define RISCV_CMD_VAL 3 +#if 1 + // for code size optimization: + #define RISCV_DELTA_7F 0x7f +#else + #define RISCV_DELTA_7F 0 +#endif + +#define RISCV_CHECK_1(v, b) \ + (((((b) - RISCV_CMD_VAL) ^ ((v) << 8)) & (0xf8000 + RISCV_CMD_VAL)) == 0) + +#if 1 + #define RISCV_CHECK_2(v, r) \ + ((((v) - ((RISCV_CMD_VAL << 12) | RISCV_REG_VAL | 8)) \ + << 18) \ + < ((r) & 0x1d)) +#else + // this branch gives larger code, because + // compilers generate larger code for big constants. + #define RISCV_CHECK_2(v, r) \ + ((((v) - ((RISCV_CMD_VAL << 12) | RISCV_REG_VAL)) \ + & ((RISCV_CMD_VAL << 12) | RISCV_REG_VAL)) \ + < ((r) & 0x1d)) +#endif + + +#define RISCV_SCAN_LOOP \ + Byte *lim; \ + size &= ~(SizeT)(RISCV_INSTR_SIZE - 1); \ + if (size <= 6) return p; \ + size -= 6; \ + lim = p + size; \ + BR_PC_INIT \ + for (;;) \ + { \ + UInt32 a, v; \ + /* Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE */ \ + for (;;) \ + { \ + if Z7_UNLIKELY(p >= lim) { return p; } \ + a = (RISCV_LOAD_VAL(p) ^ 0x10u) + 1; \ + if ((a & 0x77) == 0) break; \ + a = (RISCV_LOAD_VAL(p + RISCV_INSTR_SIZE) ^ 0x10u) + 1; \ + p += RISCV_INSTR_SIZE * 2; \ + if ((a & 0x77) == 0) \ + { \ + p -= RISCV_INSTR_SIZE; \ + if Z7_UNLIKELY(p >= lim) { return p; } \ + break; \ + } \ + } +// (xx6f ^ 10) + 1 = xx7f + 1 = xx80 : JAL +// (xxef ^ 10) + 1 = xxff + 1 = xx00 + 100 : JAL +// (xx17 ^ 10) + 1 = xx07 + 1 = xx08 : AUIPC +// (xx97 ^ 10) + 1 = xx87 + 1 = xx88 : AUIPC + +Byte * Z7_BRANCH_CONV_ENC(RISCV)(Byte *p, SizeT size, UInt32 pc) +{ + RISCV_SCAN_LOOP + v = a; + a = RISCV_GET_UI32(p); +#ifndef RISCV_USE_16BIT_LOAD + v += (UInt32)p[1] << 8; +#endif + + if ((v & 8) == 0) // JAL + { + if ((v - (0x100 /* - RISCV_DELTA_7F */)) & 0xd80) + { + p += RISCV_INSTR_SIZE; + continue; + } + { + v = ((a & 1u << 31) >> 11) + | ((a & 0x3ff << 21) >> 20) + | ((a & 1 << 20) >> 9) + | (a & 0xff << 12); + BR_CONVERT_VAL_ENC(v) + // ((v & 1) == 0) + // v: bits [1 : 20] contain offset bits +#if 0 && defined(RISCV_USE_UNALIGNED_LOAD) + a &= 0xfff; + a |= ((UInt32)(v << 23)) + | ((UInt32)(v << 7) & ((UInt32)0xff << 16)) + | ((UInt32)(v >> 5) & ((UInt32)0xf0 << 8)); + RISCV_SET_UI32(p, a) +#else // aligned +#if 0 + SetUi16a(p, (UInt16)(((v >> 5) & 0xf000) | (a & 0xfff))) +#else + p[1] = (Byte)(((v >> 13) & 0xf0) | ((a >> 8) & 0xf)); +#endif + +#if 1 && defined(Z7_CPU_FAST_BSWAP_SUPPORTED) && defined(MY_CPU_LE) + v <<= 15; + v = Z7_BSWAP32(v); + SetUi16a(p + 2, (UInt16)v) +#else + p[2] = (Byte)(v >> 9); + p[3] = (Byte)(v >> 1); +#endif +#endif // aligned + } + p += 4; + continue; + } // JAL + + { + // AUIPC + if (v & 0xe80) // (not x0) and (not x2) + { + const UInt32 b = RISCV_GET_UI32(p + 4); + if (RISCV_CHECK_1(v, b)) + { + { + const UInt32 temp = (b << 12) | (0x17 + RISCV_REG_VAL); + RISCV_SET_UI32(p, temp) + } + a &= 0xfffff000; + { +#if 1 + const int t = -1 >> 1; + if (t != -1) + a += (b >> 20) - ((b >> 19) & 0x1000); // arithmetic right shift emulation + else +#endif + a += (UInt32)((Int32)b >> 20); // arithmetic right shift (sign-extension). + } + BR_CONVERT_VAL_ENC(a) +#if 1 && defined(Z7_CPU_FAST_BSWAP_SUPPORTED) && defined(MY_CPU_LE) + a = Z7_BSWAP32(a); + RISCV_SET_UI32(p + 4, a) +#else + SetBe32(p + 4, a) +#endif + p += 8; + } + else + p += RISCV_STEP_1; + } + else + { + UInt32 r = a >> 27; + if (RISCV_CHECK_2(v, r)) + { + v = RISCV_GET_UI32(p + 4); + r = (r << 7) + 0x17 + (v & 0xfffff000); + a = (a >> 12) | (v << 20); + RISCV_SET_UI32(p, r) + RISCV_SET_UI32(p + 4, a) + p += 8; + } + else + p += RISCV_STEP_2; + } + } + } // for +} + + +Byte * Z7_BRANCH_CONV_DEC(RISCV)(Byte *p, SizeT size, UInt32 pc) +{ + RISCV_SCAN_LOOP +#ifdef RISCV_USE_16BIT_LOAD + if ((a & 8) == 0) + { +#else + v = a; + a += (UInt32)p[1] << 8; + if ((v & 8) == 0) + { +#endif + // JAL + a -= 0x100 - RISCV_DELTA_7F; + if (a & 0xd80) + { + p += RISCV_INSTR_SIZE; + continue; + } + { + const UInt32 a_old = (a + (0xef - RISCV_DELTA_7F)) & 0xfff; +#if 0 // unaligned + a = GetUi32(p); + v = (UInt32)(a >> 23) & ((UInt32)0xff << 1) + | (UInt32)(a >> 7) & ((UInt32)0xff << 9) +#elif 1 && defined(Z7_CPU_FAST_BSWAP_SUPPORTED) && defined(MY_CPU_LE) + v = GetUi16a(p + 2); + v = Z7_BSWAP32(v) >> 15 +#else + v = (UInt32)p[3] << 1 + | (UInt32)p[2] << 9 +#endif + | (UInt32)((a & 0xf000) << 5); + BR_CONVERT_VAL_DEC(v) + a = a_old + | (v << 11 & 1u << 31) + | (v << 20 & 0x3ff << 21) + | (v << 9 & 1 << 20) + | (v & 0xff << 12); + RISCV_SET_UI32(p, a) + } + p += 4; + continue; + } // JAL + + { + // AUIPC + v = a; +#if 1 && defined(RISCV_USE_UNALIGNED_LOAD) + a = GetUi32(p); +#else + a |= (UInt32)GetUi16a(p + 2) << 16; +#endif + if ((v & 0xe80) == 0) // x0/x2 + { + const UInt32 r = a >> 27; + if (RISCV_CHECK_2(v, r)) + { + UInt32 b; +#if 1 && defined(Z7_CPU_FAST_BSWAP_SUPPORTED) && defined(MY_CPU_LE) + b = RISCV_GET_UI32(p + 4); + b = Z7_BSWAP32(b); +#else + b = GetBe32(p + 4); +#endif + v = a >> 12; + BR_CONVERT_VAL_DEC(b) + a = (r << 7) + 0x17; + a += (b + 0x800) & 0xfffff000; + v |= b << 20; + RISCV_SET_UI32(p, a) + RISCV_SET_UI32(p + 4, v) + p += 8; + } + else + p += RISCV_STEP_2; + } + else + { + const UInt32 b = RISCV_GET_UI32(p + 4); + if (!RISCV_CHECK_1(v, b)) + p += RISCV_STEP_1; + else + { + v = (a & 0xfffff000) | (b >> 20); + a = (b << 12) | (0x17 + RISCV_REG_VAL); + RISCV_SET_UI32(p, a) + RISCV_SET_UI32(p + 4, v) + p += 8; + } + } + } + } // for +} diff --git a/C/Bra.h b/C/Bra.h index a4ee568..b47112c 100644 --- a/C/Bra.h +++ b/C/Bra.h @@ -1,5 +1,5 @@ /* Bra.h -- Branch converters for executables -2023-04-02 : Igor Pavlov : Public domain */ +2024-01-20 : Igor Pavlov : Public domain */ #ifndef ZIP7_INC_BRA_H #define ZIP7_INC_BRA_H @@ -8,8 +8,12 @@ EXTERN_C_BEGIN -#define Z7_BRANCH_CONV_DEC(name) z7_BranchConv_ ## name ## _Dec -#define Z7_BRANCH_CONV_ENC(name) z7_BranchConv_ ## name ## _Enc +/* #define PPC BAD_PPC_11 // for debug */ + +#define Z7_BRANCH_CONV_DEC_2(name) z7_ ## name ## _Dec +#define Z7_BRANCH_CONV_ENC_2(name) z7_ ## name ## _Enc +#define Z7_BRANCH_CONV_DEC(name) Z7_BRANCH_CONV_DEC_2(BranchConv_ ## name) +#define Z7_BRANCH_CONV_ENC(name) Z7_BRANCH_CONV_ENC_2(BranchConv_ ## name) #define Z7_BRANCH_CONV_ST_DEC(name) z7_BranchConvSt_ ## name ## _Dec #define Z7_BRANCH_CONV_ST_ENC(name) z7_BranchConvSt_ ## name ## _Enc @@ -20,19 +24,20 @@ typedef Z7_BRANCH_CONV_DECL( (*z7_Func_BranchConv)); typedef Z7_BRANCH_CONV_ST_DECL((*z7_Func_BranchConvSt)); #define Z7_BRANCH_CONV_ST_X86_STATE_INIT_VAL 0 -Z7_BRANCH_CONV_ST_DECL(Z7_BRANCH_CONV_ST_DEC(X86)); -Z7_BRANCH_CONV_ST_DECL(Z7_BRANCH_CONV_ST_ENC(X86)); +Z7_BRANCH_CONV_ST_DECL (Z7_BRANCH_CONV_ST_DEC(X86)); +Z7_BRANCH_CONV_ST_DECL (Z7_BRANCH_CONV_ST_ENC(X86)); #define Z7_BRANCH_FUNCS_DECL(name) \ -Z7_BRANCH_CONV_DECL(Z7_BRANCH_CONV_DEC(name)); \ -Z7_BRANCH_CONV_DECL(Z7_BRANCH_CONV_ENC(name)); +Z7_BRANCH_CONV_DECL (Z7_BRANCH_CONV_DEC_2(name)); \ +Z7_BRANCH_CONV_DECL (Z7_BRANCH_CONV_ENC_2(name)); -Z7_BRANCH_FUNCS_DECL(ARM64) -Z7_BRANCH_FUNCS_DECL(ARM) -Z7_BRANCH_FUNCS_DECL(ARMT) -Z7_BRANCH_FUNCS_DECL(PPC) -Z7_BRANCH_FUNCS_DECL(SPARC) -Z7_BRANCH_FUNCS_DECL(IA64) +Z7_BRANCH_FUNCS_DECL (BranchConv_ARM64) +Z7_BRANCH_FUNCS_DECL (BranchConv_ARM) +Z7_BRANCH_FUNCS_DECL (BranchConv_ARMT) +Z7_BRANCH_FUNCS_DECL (BranchConv_PPC) +Z7_BRANCH_FUNCS_DECL (BranchConv_SPARC) +Z7_BRANCH_FUNCS_DECL (BranchConv_IA64) +Z7_BRANCH_FUNCS_DECL (BranchConv_RISCV) /* These functions convert data that contain CPU instructions. @@ -49,14 +54,14 @@ and one for decoding (_Enc/_Dec postfixes in function name). In params: data : data buffer size : size of data - pc : current virtual Program Counter (Instruction Pinter) value + pc : current virtual Program Counter (Instruction Pointer) value In/Out param: state : pointer to state variable (for X86 converter only) Return: The pointer to position in (data) buffer after last byte that was processed. If the caller calls converter again, it must call it starting with that position. - But the caller is allowed to move data in buffer. so pointer to + But the caller is allowed to move data in buffer. So pointer to current processed position also will be changed for next call. Also the caller must increase internal (pc) value for next call. @@ -65,6 +70,7 @@ Each converter has some characteristics: Endian, Alignment, LookAhead. X86 little 1 4 ARMT little 2 2 + RISCV little 2 6 ARM little 4 0 ARM64 little 4 0 PPC big 4 0 diff --git a/C/Compiler.h b/C/Compiler.h index 185a52d..2a9c2b7 100644 --- a/C/Compiler.h +++ b/C/Compiler.h @@ -1,5 +1,5 @@ /* Compiler.h : Compiler specific defines and pragmas -2023-04-02 : Igor Pavlov : Public domain */ +2024-01-22 : Igor Pavlov : Public domain */ #ifndef ZIP7_INC_COMPILER_H #define ZIP7_INC_COMPILER_H @@ -25,11 +25,79 @@ #define Z7_MINGW #endif +#if defined(__LCC__) && (defined(__MCST__) || defined(__e2k__)) +#define Z7_MCST_LCC +#define Z7_MCST_LCC_VERSION (__LCC__ * 100 + __LCC_MINOR__) +#endif + +/* +#if defined(__AVX2__) \ + || defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 40900) \ + || defined(Z7_APPLE_CLANG_VERSION) && (Z7_APPLE_CLANG_VERSION >= 40600) \ + || defined(Z7_LLVM_CLANG_VERSION) && (Z7_LLVM_CLANG_VERSION >= 30100) \ + || defined(Z7_MSC_VER_ORIGINAL) && (Z7_MSC_VER_ORIGINAL >= 1800) \ + || defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1400) + #define Z7_COMPILER_AVX2_SUPPORTED + #endif +#endif +*/ + // #pragma GCC diagnostic ignored "-Wunknown-pragmas" #ifdef __clang__ // padding size of '' with 4 bytes to alignment boundary #pragma GCC diagnostic ignored "-Wpadded" + +#if defined(Z7_LLVM_CLANG_VERSION) && (__clang_major__ == 13) \ + && defined(__FreeBSD__) +// freebsd: +#pragma GCC diagnostic ignored "-Wexcess-padding" +#endif + +#if __clang_major__ >= 16 +#pragma GCC diagnostic ignored "-Wunsafe-buffer-usage" +#endif + +#if __clang_major__ == 13 +#if defined(__SIZEOF_POINTER__) && (__SIZEOF_POINTER__ == 16) +// cheri +#pragma GCC diagnostic ignored "-Wcapability-to-integer-cast" +#endif +#endif + +#if __clang_major__ == 13 + // for + #pragma GCC diagnostic ignored "-Wreserved-identifier" +#endif + +#endif // __clang__ + +#if defined(_WIN32) && defined(__clang__) && __clang_major__ >= 16 +// #pragma GCC diagnostic ignored "-Wcast-function-type-strict" +#define Z7_DIAGNOSTIC_IGNORE_CAST_FUNCTION \ + _Pragma("GCC diagnostic ignored \"-Wcast-function-type-strict\"") +#else +#define Z7_DIAGNOSTIC_IGNORE_CAST_FUNCTION +#endif + +typedef void (*Z7_void_Function)(void); +#if defined(__clang__) || defined(__GNUC__) +#define Z7_CAST_FUNC_C (Z7_void_Function) +#elif defined(_MSC_VER) && _MSC_VER > 1920 +#define Z7_CAST_FUNC_C (void *) +// #pragma warning(disable : 4191) // 'type cast': unsafe conversion from 'FARPROC' to 'void (__cdecl *)()' +#else +#define Z7_CAST_FUNC_C +#endif +/* +#if (defined(__GNUC__) && (__GNUC__ >= 8)) || defined(__clang__) + // #pragma GCC diagnostic ignored "-Wcast-function-type" +#endif +*/ +#ifdef __GNUC__ +#if defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 40000) && (Z7_GCC_VERSION < 70000) +#pragma GCC diagnostic ignored "-Wstrict-aliasing" +#endif #endif @@ -101,7 +169,8 @@ _Pragma("clang loop unroll(disable)") \ _Pragma("clang loop vectorize(disable)") #define Z7_ATTRIB_NO_VECTORIZE -#elif defined(__GNUC__) && (__GNUC__ >= 5) +#elif defined(__GNUC__) && (__GNUC__ >= 5) \ + && (!defined(Z7_MCST_LCC_VERSION) || (Z7_MCST_LCC_VERSION >= 12610)) #define Z7_ATTRIB_NO_VECTORIZE __attribute__((optimize("no-tree-vectorize"))) // __attribute__((optimize("no-unroll-loops"))); #define Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE @@ -142,15 +211,23 @@ #endif -#if (defined(Z7_CLANG_VERSION) && (Z7_CLANG_VERSION >= 36000)) -#define Z7_DIAGNOSCTIC_IGNORE_BEGIN_RESERVED_MACRO_IDENTIFIER \ +#if (defined(Z7_CLANG_VERSION) && (Z7_CLANG_VERSION >= 30600)) + +#if (Z7_CLANG_VERSION < 130000) +#define Z7_DIAGNOSTIC_IGNORE_BEGIN_RESERVED_MACRO_IDENTIFIER \ + _Pragma("GCC diagnostic push") \ + _Pragma("GCC diagnostic ignored \"-Wreserved-id-macro\"") +#else +#define Z7_DIAGNOSTIC_IGNORE_BEGIN_RESERVED_MACRO_IDENTIFIER \ _Pragma("GCC diagnostic push") \ _Pragma("GCC diagnostic ignored \"-Wreserved-macro-identifier\"") -#define Z7_DIAGNOSCTIC_IGNORE_END_RESERVED_MACRO_IDENTIFIER \ +#endif + +#define Z7_DIAGNOSTIC_IGNORE_END_RESERVED_MACRO_IDENTIFIER \ _Pragma("GCC diagnostic pop") #else -#define Z7_DIAGNOSCTIC_IGNORE_BEGIN_RESERVED_MACRO_IDENTIFIER -#define Z7_DIAGNOSCTIC_IGNORE_END_RESERVED_MACRO_IDENTIFIER +#define Z7_DIAGNOSTIC_IGNORE_BEGIN_RESERVED_MACRO_IDENTIFIER +#define Z7_DIAGNOSTIC_IGNORE_END_RESERVED_MACRO_IDENTIFIER #endif #define UNUSED_VAR(x) (void)x; diff --git a/C/CpuArch.c b/C/CpuArch.c index 33f8a3a..d51b38a 100644 --- a/C/CpuArch.c +++ b/C/CpuArch.c @@ -1,5 +1,5 @@ /* CpuArch.c -- CPU specific code -2023-05-18 : Igor Pavlov : Public domain */ +2024-03-02 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -226,7 +226,7 @@ void __declspec(naked) Z7_FASTCALL z7_x86_cpuid(UInt32 p[4], UInt32 func) DON'T remove Z7_NO_INLINE and Z7_FASTCALL for MY_cpuidex_HACK(): !!! */ static -Z7_NO_INLINE void Z7_FASTCALL MY_cpuidex_HACK(UInt32 subFunction, UInt32 func, int *CPUInfo) +Z7_NO_INLINE void Z7_FASTCALL MY_cpuidex_HACK(Int32 subFunction, Int32 func, Int32 *CPUInfo) { UNUSED_VAR(subFunction) __cpuid(CPUInfo, func); @@ -242,13 +242,13 @@ Z7_NO_INLINE #endif void Z7_FASTCALL z7_x86_cpuid(UInt32 p[4], UInt32 func) { - MY_cpuidex((int *)p, (int)func, 0); + MY_cpuidex((Int32 *)p, (Int32)func, 0); } Z7_NO_INLINE UInt32 Z7_FASTCALL z7_x86_cpuid_GetMaxFunc(void) { - int a[4]; + Int32 a[4]; MY_cpuidex(a, 0, 0); return a[0]; } @@ -384,7 +384,7 @@ BoolInt CPU_IsSupported_CMOV(void) UInt32 a[4]; if (!x86cpuid_Func_1(&a[0])) return 0; - return (a[3] >> 15) & 1; + return (BoolInt)(a[3] >> 15) & 1; } BoolInt CPU_IsSupported_SSE(void) @@ -393,7 +393,7 @@ BoolInt CPU_IsSupported_SSE(void) CHECK_SYS_SSE_SUPPORT if (!x86cpuid_Func_1(&a[0])) return 0; - return (a[3] >> 25) & 1; + return (BoolInt)(a[3] >> 25) & 1; } BoolInt CPU_IsSupported_SSE2(void) @@ -402,7 +402,7 @@ BoolInt CPU_IsSupported_SSE2(void) CHECK_SYS_SSE_SUPPORT if (!x86cpuid_Func_1(&a[0])) return 0; - return (a[3] >> 26) & 1; + return (BoolInt)(a[3] >> 26) & 1; } #endif @@ -419,17 +419,17 @@ static UInt32 x86cpuid_Func_1_ECX(void) BoolInt CPU_IsSupported_AES(void) { - return (x86cpuid_Func_1_ECX() >> 25) & 1; + return (BoolInt)(x86cpuid_Func_1_ECX() >> 25) & 1; } BoolInt CPU_IsSupported_SSSE3(void) { - return (x86cpuid_Func_1_ECX() >> 9) & 1; + return (BoolInt)(x86cpuid_Func_1_ECX() >> 9) & 1; } BoolInt CPU_IsSupported_SSE41(void) { - return (x86cpuid_Func_1_ECX() >> 19) & 1; + return (BoolInt)(x86cpuid_Func_1_ECX() >> 19) & 1; } BoolInt CPU_IsSupported_SHA(void) @@ -441,7 +441,7 @@ BoolInt CPU_IsSupported_SHA(void) { UInt32 d[4]; z7_x86_cpuid(d, 7); - return (d[1] >> 29) & 1; + return (BoolInt)(d[1] >> 29) & 1; } } @@ -640,8 +640,8 @@ BoolInt CPU_IsSupported_AVX(void) const UInt32 bm = (UInt32)x86_xgetbv_0(MY_XCR_XFEATURE_ENABLED_MASK); // printf("\n=== XGetBV=%d\n", bm); return 1 - & (bm >> 1) // SSE state is supported (set by OS) for storing/restoring - & (bm >> 2); // AVX state is supported (set by OS) for storing/restoring + & (BoolInt)(bm >> 1) // SSE state is supported (set by OS) for storing/restoring + & (BoolInt)(bm >> 2); // AVX state is supported (set by OS) for storing/restoring } // since Win7SP1: we can use GetEnabledXStateFeatures(); } @@ -658,10 +658,29 @@ BoolInt CPU_IsSupported_AVX2(void) z7_x86_cpuid(d, 7); // printf("\ncpuid(7): ebx=%8x ecx=%8x\n", d[1], d[2]); return 1 - & (d[1] >> 5); // avx2 + & (BoolInt)(d[1] >> 5); // avx2 } } +/* +// fix it: +BoolInt CPU_IsSupported_AVX512F_AVX512VL(void) +{ + if (!CPU_IsSupported_AVX()) + return False; + if (z7_x86_cpuid_GetMaxFunc() < 7) + return False; + { + UInt32 d[4]; + z7_x86_cpuid(d, 7); + // printf("\ncpuid(7): ebx=%8x ecx=%8x\n", d[1], d[2]); + return 1 + & (BoolInt)(d[1] >> 16) // avx512-f + & (BoolInt)(d[1] >> 31); // avx512-Vl + } +} +*/ + BoolInt CPU_IsSupported_VAES_AVX2(void) { if (!CPU_IsSupported_AVX()) @@ -673,9 +692,9 @@ BoolInt CPU_IsSupported_VAES_AVX2(void) z7_x86_cpuid(d, 7); // printf("\ncpuid(7): ebx=%8x ecx=%8x\n", d[1], d[2]); return 1 - & (d[1] >> 5) // avx2 + & (BoolInt)(d[1] >> 5) // avx2 // & (d[1] >> 31) // avx512vl - & (d[2] >> 9); // vaes // VEX-256/EVEX + & (BoolInt)(d[2] >> 9); // vaes // VEX-256/EVEX } } @@ -688,7 +707,7 @@ BoolInt CPU_IsSupported_PageGB(void) if (d[0] < 0x80000001) return False; z7_x86_cpuid(d, 0x80000001); - return (d[3] >> 26) & 1; + return (BoolInt)(d[3] >> 26) & 1; } } @@ -760,32 +779,65 @@ BoolInt CPU_IsSupported_AES (void) { return APPLE_CRYPTO_SUPPORT_VAL; } #else // __APPLE__ -#include +#if defined(__GLIBC__) && (__GLIBC__ * 100 + __GLIBC_MINOR__ >= 216) + #define Z7_GETAUXV_AVAILABLE +#else +// #pragma message("=== is not NEW GLIBC === ") + #if defined __has_include + #if __has_include () +// #pragma message("=== sys/auxv.h is avail=== ") + #define Z7_GETAUXV_AVAILABLE + #endif + #endif +#endif +#ifdef Z7_GETAUXV_AVAILABLE +// #pragma message("=== Z7_GETAUXV_AVAILABLE === ") +#include #define USE_HWCAP +#endif #ifdef USE_HWCAP +#if defined(__FreeBSD__) +static unsigned long MY_getauxval(int aux) +{ + unsigned long val; + if (elf_aux_info(aux, &val, sizeof(val))) + return 0; + return val; +} +#else +#define MY_getauxval getauxval + #if defined __has_include + #if __has_include () #include + #endif + #endif +#endif #define MY_HWCAP_CHECK_FUNC_2(name1, name2) \ - BoolInt CPU_IsSupported_ ## name1() { return (getauxval(AT_HWCAP) & (HWCAP_ ## name2)) ? 1 : 0; } + BoolInt CPU_IsSupported_ ## name1(void) { return (MY_getauxval(AT_HWCAP) & (HWCAP_ ## name2)); } #ifdef MY_CPU_ARM64 #define MY_HWCAP_CHECK_FUNC(name) \ MY_HWCAP_CHECK_FUNC_2(name, name) +#if 1 || defined(__ARM_NEON) + BoolInt CPU_IsSupported_NEON(void) { return True; } +#else MY_HWCAP_CHECK_FUNC_2(NEON, ASIMD) +#endif // MY_HWCAP_CHECK_FUNC (ASIMD) #elif defined(MY_CPU_ARM) #define MY_HWCAP_CHECK_FUNC(name) \ - BoolInt CPU_IsSupported_ ## name() { return (getauxval(AT_HWCAP2) & (HWCAP2_ ## name)) ? 1 : 0; } + BoolInt CPU_IsSupported_ ## name(void) { return (MY_getauxval(AT_HWCAP2) & (HWCAP2_ ## name)); } MY_HWCAP_CHECK_FUNC_2(NEON, NEON) #endif #else // USE_HWCAP #define MY_HWCAP_CHECK_FUNC(name) \ - BoolInt CPU_IsSupported_ ## name() { return 0; } + BoolInt CPU_IsSupported_ ## name(void) { return 0; } MY_HWCAP_CHECK_FUNC(NEON) #endif // USE_HWCAP diff --git a/C/CpuArch.h b/C/CpuArch.h index 8e5d8a5..dfc68f1 100644 --- a/C/CpuArch.h +++ b/C/CpuArch.h @@ -1,5 +1,5 @@ /* CpuArch.h -- CPU specific code -2023-04-02 : Igor Pavlov : Public domain */ +2024-05-13 : Igor Pavlov : Public domain */ #ifndef ZIP7_INC_CPU_ARCH_H #define ZIP7_INC_CPU_ARCH_H @@ -20,6 +20,7 @@ MY_CPU_64BIT means that processor can work with 64-bit registers. MY_CPU_64BIT doesn't mean that (sizeof(void *) == 8) */ +#if !defined(_M_ARM64EC) #if defined(_M_X64) \ || defined(_M_AMD64) \ || defined(__x86_64__) \ @@ -35,6 +36,7 @@ MY_CPU_64BIT means that processor can work with 64-bit registers. #endif #define MY_CPU_64BIT #endif +#endif #if defined(_M_IX86) \ @@ -47,17 +49,26 @@ MY_CPU_64BIT means that processor can work with 64-bit registers. #if defined(_M_ARM64) \ + || defined(_M_ARM64EC) \ || defined(__AARCH64EL__) \ || defined(__AARCH64EB__) \ || defined(__aarch64__) #define MY_CPU_ARM64 - #ifdef __ILP32__ +#if defined(__ILP32__) \ + || defined(__SIZEOF_POINTER__) && (__SIZEOF_POINTER__ == 4) #define MY_CPU_NAME "arm64-32" #define MY_CPU_SIZEOF_POINTER 4 - #else +#elif defined(__SIZEOF_POINTER__) && (__SIZEOF_POINTER__ == 16) + #define MY_CPU_NAME "arm64-128" + #define MY_CPU_SIZEOF_POINTER 16 +#else +#if defined(_M_ARM64EC) + #define MY_CPU_NAME "arm64ec" +#else #define MY_CPU_NAME "arm64" +#endif #define MY_CPU_SIZEOF_POINTER 8 - #endif +#endif #define MY_CPU_64BIT #endif @@ -133,8 +144,36 @@ MY_CPU_64BIT means that processor can work with 64-bit registers. #endif +#if defined(__sparc__) \ + || defined(__sparc) + #define MY_CPU_SPARC + #if defined(__LP64__) \ + || defined(_LP64) \ + || defined(__SIZEOF_POINTER__) && (__SIZEOF_POINTER__ == 8) + #define MY_CPU_NAME "sparcv9" + #define MY_CPU_SIZEOF_POINTER 8 + #define MY_CPU_64BIT + #elif defined(__sparc_v9__) \ + || defined(__sparcv9) + #define MY_CPU_64BIT + #if defined(__SIZEOF_POINTER__) && (__SIZEOF_POINTER__ == 4) + #define MY_CPU_NAME "sparcv9-32" + #else + #define MY_CPU_NAME "sparcv9m" + #endif + #elif defined(__sparc_v8__) \ + || defined(__sparcv8) + #define MY_CPU_NAME "sparcv8" + #define MY_CPU_SIZEOF_POINTER 4 + #else + #define MY_CPU_NAME "sparc" + #endif +#endif + + #if defined(__riscv) \ || defined(__riscv__) + #define MY_CPU_RISCV #if __riscv_xlen == 32 #define MY_CPU_NAME "riscv32" #elif __riscv_xlen == 64 @@ -145,6 +184,39 @@ MY_CPU_64BIT means that processor can work with 64-bit registers. #endif +#if defined(__loongarch__) + #define MY_CPU_LOONGARCH + #if defined(__loongarch64) || defined(__loongarch_grlen) && (__loongarch_grlen == 64) + #define MY_CPU_64BIT + #endif + #if defined(__loongarch64) + #define MY_CPU_NAME "loongarch64" + #define MY_CPU_LOONGARCH64 + #else + #define MY_CPU_NAME "loongarch" + #endif +#endif + + +// #undef MY_CPU_NAME +// #undef MY_CPU_SIZEOF_POINTER +// #define __e2k__ +// #define __SIZEOF_POINTER__ 4 +#if defined(__e2k__) + #define MY_CPU_E2K + #if defined(__ILP32__) || defined(__SIZEOF_POINTER__) && (__SIZEOF_POINTER__ == 4) + #define MY_CPU_NAME "e2k-32" + #define MY_CPU_SIZEOF_POINTER 4 + #else + #define MY_CPU_NAME "e2k" + #if defined(__LP64__) || defined(__SIZEOF_POINTER__) && (__SIZEOF_POINTER__ == 8) + #define MY_CPU_SIZEOF_POINTER 8 + #endif + #endif + #define MY_CPU_64BIT +#endif + + #if defined(MY_CPU_X86) || defined(MY_CPU_AMD64) #define MY_CPU_X86_OR_AMD64 #endif @@ -175,6 +247,7 @@ MY_CPU_64BIT means that processor can work with 64-bit registers. || defined(MY_CPU_ARM_LE) \ || defined(MY_CPU_ARM64_LE) \ || defined(MY_CPU_IA64_LE) \ + || defined(_LITTLE_ENDIAN) \ || defined(__LITTLE_ENDIAN__) \ || defined(__ARMEL__) \ || defined(__THUMBEL__) \ @@ -251,6 +324,7 @@ MY_CPU_64BIT means that processor can work with 64-bit registers. #ifndef MY_CPU_NAME + // #define MY_CPU_IS_UNKNOWN #ifdef MY_CPU_LE #define MY_CPU_NAME "LE" #elif defined(MY_CPU_BE) @@ -295,9 +369,19 @@ MY_CPU_64BIT means that processor can work with 64-bit registers. #define Z7_BSWAP64(v) _byteswap_uint64(v) #define Z7_CPU_FAST_BSWAP_SUPPORTED -#elif (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3))) \ - || (defined(__clang__) && Z7_has_builtin(__builtin_bswap16)) - +/* GCC can generate slow code that calls function for __builtin_bswap32() for: + - GCC for RISCV, if Zbb extension is not used. + - GCC for SPARC. + The code from CLANG for SPARC also is not fastest. + So we don't define Z7_CPU_FAST_BSWAP_SUPPORTED in some cases. +*/ +#elif (!defined(MY_CPU_RISCV) || defined (__riscv_zbb)) \ + && !defined(MY_CPU_SPARC) \ + && ( \ + (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3))) \ + || (defined(__clang__) && Z7_has_builtin(__builtin_bswap16)) \ + ) + #define Z7_BSWAP16(v) __builtin_bswap16(v) #define Z7_BSWAP32(v) __builtin_bswap32(v) #define Z7_BSWAP64(v) __builtin_bswap64(v) @@ -329,13 +413,48 @@ MY_CPU_64BIT means that processor can work with 64-bit registers. #ifdef MY_CPU_LE #if defined(MY_CPU_X86_OR_AMD64) \ - || defined(MY_CPU_ARM64) + || defined(MY_CPU_ARM64) \ + || defined(MY_CPU_RISCV) && defined(__riscv_misaligned_fast) \ + || defined(MY_CPU_E2K) && defined(__iset__) && (__iset__ >= 6) #define MY_CPU_LE_UNALIGN #define MY_CPU_LE_UNALIGN_64 #elif defined(__ARM_FEATURE_UNALIGNED) - /* gcc9 for 32-bit arm can use LDRD instruction that requires 32-bit alignment. - So we can't use unaligned 64-bit operations. */ - #define MY_CPU_LE_UNALIGN +/* === ALIGNMENT on 32-bit arm and LDRD/STRD/LDM/STM instructions. + Description of problems: +problem-1 : 32-bit ARM architecture: + multi-access (pair of 32-bit accesses) instructions (LDRD/STRD/LDM/STM) + require 32-bit (WORD) alignment (by 32-bit ARM architecture). + So there is "Alignment fault exception", if data is not aligned for 32-bit. + +problem-2 : 32-bit kernels and arm64 kernels: + 32-bit linux kernels provide fixup for these "paired" instruction "Alignment fault exception". + So unaligned paired-access instructions work via exception handler in kernel in 32-bit linux. + + But some arm64 kernels do not handle these faults in 32-bit programs. + So we have unhandled exception for such instructions. + Probably some new arm64 kernels have fixed it, and unaligned + paired-access instructions work in new kernels? + +problem-3 : compiler for 32-bit arm: + Compilers use LDRD/STRD/LDM/STM for UInt64 accesses + and for another cases where two 32-bit accesses are fused + to one multi-access instruction. + So UInt64 variables must be aligned for 32-bit, and each + 32-bit access must be aligned for 32-bit, if we want to + avoid "Alignment fault" exception (handled or unhandled). + +problem-4 : performace: + Even if unaligned access is handled by kernel, it will be slow. + So if we allow unaligned access, we can get fast unaligned + single-access, and slow unaligned paired-access. + + We don't allow unaligned access on 32-bit arm, because compiler + genarates paired-access instructions that require 32-bit alignment, + and some arm64 kernels have no handler for these instructions. + Also unaligned paired-access instructions will be slow, if kernel handles them. +*/ + // it must be disabled: + // #define MY_CPU_LE_UNALIGN #endif #endif @@ -439,6 +558,7 @@ MY_CPU_64BIT means that processor can work with 64-bit registers. #if defined(MY_CPU_BE) +#define GetBe64a(p) (*(const UInt64 *)(const void *)(p)) #define GetBe32a(p) (*(const UInt32 *)(const void *)(p)) #define GetBe16a(p) (*(const UInt16 *)(const void *)(p)) #define SetBe32a(p, v) { *(UInt32 *)(void *)(p) = (v); } @@ -456,6 +576,7 @@ MY_CPU_64BIT means that processor can work with 64-bit registers. #define SetUi32a(p, v) { *(UInt32 *)(void *)(p) = (v); } #define SetUi16a(p, v) { *(UInt16 *)(void *)(p) = (v); } +#define GetBe64a(p) GetBe64(p) #define GetBe32a(p) GetBe32(p) #define GetBe16a(p) GetBe16(p) #define SetBe32a(p, v) SetBe32(p, v) @@ -486,6 +607,7 @@ UInt32 Z7_FASTCALL z7_x86_cpuid_GetMaxFunc(void); BoolInt CPU_IsSupported_AES(void); BoolInt CPU_IsSupported_AVX(void); BoolInt CPU_IsSupported_AVX2(void); +// BoolInt CPU_IsSupported_AVX512F_AVX512VL(void); BoolInt CPU_IsSupported_VAES_AVX2(void); BoolInt CPU_IsSupported_CMOV(void); BoolInt CPU_IsSupported_SSE(void); diff --git a/C/DllSecur.c b/C/DllSecur.c index 02a0f97..bbbfc0a 100644 --- a/C/DllSecur.c +++ b/C/DllSecur.c @@ -1,5 +1,5 @@ /* DllSecur.c -- DLL loading security -2023-04-02 : Igor Pavlov : Public domain */ +2023-12-03 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -11,19 +11,7 @@ #ifndef UNDER_CE -#if (defined(__GNUC__) && (__GNUC__ >= 8)) || defined(__clang__) - // #pragma GCC diagnostic ignored "-Wcast-function-type" -#endif - -#if defined(__clang__) || defined(__GNUC__) -typedef void (*Z7_voidFunction)(void); -#define MY_CAST_FUNC (Z7_voidFunction) -#elif defined(_MSC_VER) && _MSC_VER > 1920 -#define MY_CAST_FUNC (void *) -// #pragma warning(disable : 4191) // 'type cast': unsafe conversion from 'FARPROC' to 'void (__cdecl *)()' -#else -#define MY_CAST_FUNC -#endif +Z7_DIAGNOSTIC_IGNORE_CAST_FUNCTION typedef BOOL (WINAPI *Func_SetDefaultDllDirectories)(DWORD DirectoryFlags); @@ -61,7 +49,7 @@ static const char * const g_Dlls = if ((UInt16)GetVersion() != 6) { \ const \ Func_SetDefaultDllDirectories setDllDirs = \ - (Func_SetDefaultDllDirectories) MY_CAST_FUNC GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")), \ + (Func_SetDefaultDllDirectories) Z7_CAST_FUNC_C GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")), \ "SetDefaultDllDirectories"); \ if (setDllDirs) if (setDllDirs(MY_LOAD_LIBRARY_SEARCH_SYSTEM32 | MY_LOAD_LIBRARY_SEARCH_USER_DIRS)) return; } diff --git a/C/HuffEnc.c b/C/HuffEnc.c index 3dc1e39..996da30 100644 --- a/C/HuffEnc.c +++ b/C/HuffEnc.c @@ -1,5 +1,5 @@ /* HuffEnc.c -- functions for Huffman encoding -2023-03-04 : Igor Pavlov : Public domain */ +2023-09-07 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -8,7 +8,7 @@ #define kMaxLen 16 #define NUM_BITS 10 -#define MASK (((unsigned)1 << NUM_BITS) - 1) +#define MASK ((1u << NUM_BITS) - 1) #define NUM_COUNTERS 64 diff --git a/C/LzFind.c b/C/LzFind.c index 0fbd5aa..1ce4046 100644 --- a/C/LzFind.c +++ b/C/LzFind.c @@ -1,5 +1,5 @@ /* LzFind.c -- Match finder for LZ algorithms -2023-03-14 : Igor Pavlov : Public domain */ +2024-03-01 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -108,9 +108,15 @@ static int LzInWindow_Create2(CMatchFinder *p, UInt32 blockSize, ISzAllocPtr all return (p->bufBase != NULL); } -static const Byte *MatchFinder_GetPointerToCurrentPos(CMatchFinder *p) { return p->buffer; } +static const Byte *MatchFinder_GetPointerToCurrentPos(void *p) +{ + return ((CMatchFinder *)p)->buffer; +} -static UInt32 MatchFinder_GetNumAvailableBytes(CMatchFinder *p) { return GET_AVAIL_BYTES(p); } +static UInt32 MatchFinder_GetNumAvailableBytes(void *p) +{ + return GET_AVAIL_BYTES((CMatchFinder *)p); +} Z7_NO_INLINE @@ -571,8 +577,9 @@ void MatchFinder_Init_4(CMatchFinder *p) #define CYC_TO_POS_OFFSET 0 // #define CYC_TO_POS_OFFSET 1 // for debug -void MatchFinder_Init(CMatchFinder *p) +void MatchFinder_Init(void *_p) { + CMatchFinder *p = (CMatchFinder *)_p; MatchFinder_Init_HighHash(p); MatchFinder_Init_LowHash(p); MatchFinder_Init_4(p); @@ -607,16 +614,16 @@ void MatchFinder_Init(CMatchFinder *p) #endif #endif -// #elif defined(MY_CPU_ARM_OR_ARM64) -#elif defined(MY_CPU_ARM64) +#elif defined(MY_CPU_ARM64) \ + /* || (defined(__ARM_ARCH) && (__ARM_ARCH >= 7)) */ - #if defined(__clang__) && (__clang_major__ >= 8) \ - || defined(__GNUC__) && (__GNUC__ >= 8) + #if defined(Z7_CLANG_VERSION) && (Z7_CLANG_VERSION >= 30800) \ + || defined(__GNUC__) && (__GNUC__ >= 6) #define USE_LZFIND_SATUR_SUB_128 #ifdef MY_CPU_ARM64 // #define LZFIND_ATTRIB_SSE41 __attribute__((__target__(""))) #else - // #define LZFIND_ATTRIB_SSE41 __attribute__((__target__("fpu=crypto-neon-fp-armv8"))) + #define LZFIND_ATTRIB_SSE41 __attribute__((__target__("fpu=neon"))) #endif #elif defined(_MSC_VER) @@ -625,7 +632,7 @@ void MatchFinder_Init(CMatchFinder *p) #endif #endif - #if defined(_MSC_VER) && defined(MY_CPU_ARM64) + #if defined(Z7_MSC_VER_ORIGINAL) && defined(MY_CPU_ARM64) #include #else #include @@ -1082,9 +1089,11 @@ static void SkipMatchesSpec(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const #define MOVE_POS \ - ++p->cyclicBufferPos; \ + p->cyclicBufferPos++; \ p->buffer++; \ - { const UInt32 pos1 = p->pos + 1; p->pos = pos1; if (pos1 == p->posLimit) MatchFinder_CheckLimits(p); } + { const UInt32 pos1 = p->pos + 1; \ + p->pos = pos1; \ + if (pos1 == p->posLimit) MatchFinder_CheckLimits(p); } #define MOVE_POS_RET MOVE_POS return distances; @@ -1103,20 +1112,26 @@ static void MatchFinder_MovePos(CMatchFinder *p) } #define GET_MATCHES_HEADER2(minLen, ret_op) \ - unsigned lenLimit; UInt32 hv; const Byte *cur; UInt32 curMatch; \ - lenLimit = (unsigned)p->lenLimit; { if (lenLimit < minLen) { MatchFinder_MovePos(p); ret_op; }} \ + UInt32 hv; const Byte *cur; UInt32 curMatch; \ + UInt32 lenLimit = p->lenLimit; \ + if (lenLimit < minLen) { MatchFinder_MovePos(p); ret_op; } \ cur = p->buffer; #define GET_MATCHES_HEADER(minLen) GET_MATCHES_HEADER2(minLen, return distances) -#define SKIP_HEADER(minLen) do { GET_MATCHES_HEADER2(minLen, continue) +#define SKIP_HEADER(minLen) \ + do { GET_MATCHES_HEADER2(minLen, continue) -#define MF_PARAMS(p) lenLimit, curMatch, p->pos, p->buffer, p->son, p->cyclicBufferPos, p->cyclicBufferSize, p->cutValue +#define MF_PARAMS(p) lenLimit, curMatch, p->pos, p->buffer, p->son, \ + p->cyclicBufferPos, p->cyclicBufferSize, p->cutValue -#define SKIP_FOOTER SkipMatchesSpec(MF_PARAMS(p)); MOVE_POS } while (--num); +#define SKIP_FOOTER \ + SkipMatchesSpec(MF_PARAMS(p)); \ + MOVE_POS \ + } while (--num); #define GET_MATCHES_FOOTER_BASE(_maxLen_, func) \ - distances = func(MF_PARAMS(p), \ - distances, (UInt32)_maxLen_); MOVE_POS_RET + distances = func(MF_PARAMS(p), distances, (UInt32)_maxLen_); \ + MOVE_POS_RET #define GET_MATCHES_FOOTER_BT(_maxLen_) \ GET_MATCHES_FOOTER_BASE(_maxLen_, GetMatchesSpec1) @@ -1133,8 +1148,9 @@ static void MatchFinder_MovePos(CMatchFinder *p) for (; c != lim; c++) if (*(c + diff) != *c) break; \ maxLen = (unsigned)(c - cur); } -static UInt32* Bt2_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) +static UInt32* Bt2_MatchFinder_GetMatches(void *_p, UInt32 *distances) { + CMatchFinder *p = (CMatchFinder *)_p; GET_MATCHES_HEADER(2) HASH2_CALC curMatch = p->hash[hv]; @@ -1158,8 +1174,9 @@ UInt32* Bt3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) mmm = pos; -static UInt32* Bt3_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) +static UInt32* Bt3_MatchFinder_GetMatches(void *_p, UInt32 *distances) { + CMatchFinder *p = (CMatchFinder *)_p; UInt32 mmm; UInt32 h2, d2, pos; unsigned maxLen; @@ -1199,8 +1216,9 @@ static UInt32* Bt3_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) } -static UInt32* Bt4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) +static UInt32* Bt4_MatchFinder_GetMatches(void *_p, UInt32 *distances) { + CMatchFinder *p = (CMatchFinder *)_p; UInt32 mmm; UInt32 h2, h3, d2, d3, pos; unsigned maxLen; @@ -1267,10 +1285,12 @@ static UInt32* Bt4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) } -static UInt32* Bt5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) +static UInt32* Bt5_MatchFinder_GetMatches(void *_p, UInt32 *distances) { + CMatchFinder *p = (CMatchFinder *)_p; UInt32 mmm; - UInt32 h2, h3, d2, d3, maxLen, pos; + UInt32 h2, h3, d2, d3, pos; + unsigned maxLen; UInt32 *hash; GET_MATCHES_HEADER(5) @@ -1339,8 +1359,9 @@ static UInt32* Bt5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) } -static UInt32* Hc4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) +static UInt32* Hc4_MatchFinder_GetMatches(void *_p, UInt32 *distances) { + CMatchFinder *p = (CMatchFinder *)_p; UInt32 mmm; UInt32 h2, h3, d2, d3, pos; unsigned maxLen; @@ -1407,10 +1428,12 @@ static UInt32* Hc4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) } -static UInt32 * Hc5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) +static UInt32 * Hc5_MatchFinder_GetMatches(void *_p, UInt32 *distances) { + CMatchFinder *p = (CMatchFinder *)_p; UInt32 mmm; - UInt32 h2, h3, d2, d3, maxLen, pos; + UInt32 h2, h3, d2, d3, pos; + unsigned maxLen; UInt32 *hash; GET_MATCHES_HEADER(5) @@ -1466,7 +1489,7 @@ static UInt32 * Hc5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) if (*(cur - d2 + 3) != cur[3]) break; UPDATE_maxLen - distances[-2] = maxLen; + distances[-2] = (UInt32)maxLen; if (maxLen == lenLimit) { p->son[p->cyclicBufferPos] = curMatch; @@ -1489,8 +1512,9 @@ UInt32* Hc3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) } -static void Bt2_MatchFinder_Skip(CMatchFinder *p, UInt32 num) +static void Bt2_MatchFinder_Skip(void *_p, UInt32 num) { + CMatchFinder *p = (CMatchFinder *)_p; SKIP_HEADER(2) { HASH2_CALC @@ -1511,8 +1535,9 @@ void Bt3Zip_MatchFinder_Skip(CMatchFinder *p, UInt32 num) SKIP_FOOTER } -static void Bt3_MatchFinder_Skip(CMatchFinder *p, UInt32 num) +static void Bt3_MatchFinder_Skip(void *_p, UInt32 num) { + CMatchFinder *p = (CMatchFinder *)_p; SKIP_HEADER(3) { UInt32 h2; @@ -1526,8 +1551,9 @@ static void Bt3_MatchFinder_Skip(CMatchFinder *p, UInt32 num) SKIP_FOOTER } -static void Bt4_MatchFinder_Skip(CMatchFinder *p, UInt32 num) +static void Bt4_MatchFinder_Skip(void *_p, UInt32 num) { + CMatchFinder *p = (CMatchFinder *)_p; SKIP_HEADER(4) { UInt32 h2, h3; @@ -1542,8 +1568,9 @@ static void Bt4_MatchFinder_Skip(CMatchFinder *p, UInt32 num) SKIP_FOOTER } -static void Bt5_MatchFinder_Skip(CMatchFinder *p, UInt32 num) +static void Bt5_MatchFinder_Skip(void *_p, UInt32 num) { + CMatchFinder *p = (CMatchFinder *)_p; SKIP_HEADER(5) { UInt32 h2, h3; @@ -1589,8 +1616,9 @@ static void Bt5_MatchFinder_Skip(CMatchFinder *p, UInt32 num) }} while(num); \ -static void Hc4_MatchFinder_Skip(CMatchFinder *p, UInt32 num) +static void Hc4_MatchFinder_Skip(void *_p, UInt32 num) { + CMatchFinder *p = (CMatchFinder *)_p; HC_SKIP_HEADER(4) UInt32 h2, h3; @@ -1604,8 +1632,9 @@ static void Hc4_MatchFinder_Skip(CMatchFinder *p, UInt32 num) } -static void Hc5_MatchFinder_Skip(CMatchFinder *p, UInt32 num) +static void Hc5_MatchFinder_Skip(void *_p, UInt32 num) { + CMatchFinder *p = (CMatchFinder *)_p; HC_SKIP_HEADER(5) UInt32 h2, h3; @@ -1634,41 +1663,41 @@ void Hc3Zip_MatchFinder_Skip(CMatchFinder *p, UInt32 num) void MatchFinder_CreateVTable(CMatchFinder *p, IMatchFinder2 *vTable) { - vTable->Init = (Mf_Init_Func)MatchFinder_Init; - vTable->GetNumAvailableBytes = (Mf_GetNumAvailableBytes_Func)MatchFinder_GetNumAvailableBytes; - vTable->GetPointerToCurrentPos = (Mf_GetPointerToCurrentPos_Func)MatchFinder_GetPointerToCurrentPos; + vTable->Init = MatchFinder_Init; + vTable->GetNumAvailableBytes = MatchFinder_GetNumAvailableBytes; + vTable->GetPointerToCurrentPos = MatchFinder_GetPointerToCurrentPos; if (!p->btMode) { if (p->numHashBytes <= 4) { - vTable->GetMatches = (Mf_GetMatches_Func)Hc4_MatchFinder_GetMatches; - vTable->Skip = (Mf_Skip_Func)Hc4_MatchFinder_Skip; + vTable->GetMatches = Hc4_MatchFinder_GetMatches; + vTable->Skip = Hc4_MatchFinder_Skip; } else { - vTable->GetMatches = (Mf_GetMatches_Func)Hc5_MatchFinder_GetMatches; - vTable->Skip = (Mf_Skip_Func)Hc5_MatchFinder_Skip; + vTable->GetMatches = Hc5_MatchFinder_GetMatches; + vTable->Skip = Hc5_MatchFinder_Skip; } } else if (p->numHashBytes == 2) { - vTable->GetMatches = (Mf_GetMatches_Func)Bt2_MatchFinder_GetMatches; - vTable->Skip = (Mf_Skip_Func)Bt2_MatchFinder_Skip; + vTable->GetMatches = Bt2_MatchFinder_GetMatches; + vTable->Skip = Bt2_MatchFinder_Skip; } else if (p->numHashBytes == 3) { - vTable->GetMatches = (Mf_GetMatches_Func)Bt3_MatchFinder_GetMatches; - vTable->Skip = (Mf_Skip_Func)Bt3_MatchFinder_Skip; + vTable->GetMatches = Bt3_MatchFinder_GetMatches; + vTable->Skip = Bt3_MatchFinder_Skip; } else if (p->numHashBytes == 4) { - vTable->GetMatches = (Mf_GetMatches_Func)Bt4_MatchFinder_GetMatches; - vTable->Skip = (Mf_Skip_Func)Bt4_MatchFinder_Skip; + vTable->GetMatches = Bt4_MatchFinder_GetMatches; + vTable->Skip = Bt4_MatchFinder_Skip; } else { - vTable->GetMatches = (Mf_GetMatches_Func)Bt5_MatchFinder_GetMatches; - vTable->Skip = (Mf_Skip_Func)Bt5_MatchFinder_Skip; + vTable->GetMatches = Bt5_MatchFinder_GetMatches; + vTable->Skip = Bt5_MatchFinder_Skip; } } diff --git a/C/LzFind.h b/C/LzFind.h index a3f72c9..67e8a6e 100644 --- a/C/LzFind.h +++ b/C/LzFind.h @@ -1,5 +1,5 @@ /* LzFind.h -- Match finder for LZ algorithms -2023-03-04 : Igor Pavlov : Public domain */ +2024-01-22 : Igor Pavlov : Public domain */ #ifndef ZIP7_INC_LZ_FIND_H #define ZIP7_INC_LZ_FIND_H @@ -144,7 +144,8 @@ void MatchFinder_CreateVTable(CMatchFinder *p, IMatchFinder2 *vTable); void MatchFinder_Init_LowHash(CMatchFinder *p); void MatchFinder_Init_HighHash(CMatchFinder *p); void MatchFinder_Init_4(CMatchFinder *p); -void MatchFinder_Init(CMatchFinder *p); +// void MatchFinder_Init(CMatchFinder *p); +void MatchFinder_Init(void *p); UInt32* Bt3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances); UInt32* Hc3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances); diff --git a/C/LzFindMt.c b/C/LzFindMt.c index 5253e6e..ac9d59d 100644 --- a/C/LzFindMt.c +++ b/C/LzFindMt.c @@ -1,5 +1,5 @@ /* LzFindMt.c -- multithreaded Match finder for LZ algorithms -2023-04-02 : Igor Pavlov : Public domain */ +2024-01-22 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -94,7 +94,7 @@ static void MtSync_Construct(CMtSync *p) } -#define DEBUG_BUFFER_LOCK // define it to debug lock state +// #define DEBUG_BUFFER_LOCK // define it to debug lock state #ifdef DEBUG_BUFFER_LOCK #include @@ -877,8 +877,9 @@ SRes MatchFinderMt_InitMt(CMatchFinderMt *p) } -static void MatchFinderMt_Init(CMatchFinderMt *p) +static void MatchFinderMt_Init(void *_p) { + CMatchFinderMt *p = (CMatchFinderMt *)_p; CMatchFinder *mf = MF(p); p->btBufPos = @@ -981,8 +982,9 @@ static UInt32 MatchFinderMt_GetNextBlock_Bt(CMatchFinderMt *p) -static const Byte * MatchFinderMt_GetPointerToCurrentPos(CMatchFinderMt *p) +static const Byte * MatchFinderMt_GetPointerToCurrentPos(void *_p) { + CMatchFinderMt *p = (CMatchFinderMt *)_p; return p->pointerToCurPos; } @@ -990,8 +992,9 @@ static const Byte * MatchFinderMt_GetPointerToCurrentPos(CMatchFinderMt *p) #define GET_NEXT_BLOCK_IF_REQUIRED if (p->btBufPos == p->btBufPosLimit) MatchFinderMt_GetNextBlock_Bt(p); -static UInt32 MatchFinderMt_GetNumAvailableBytes(CMatchFinderMt *p) +static UInt32 MatchFinderMt_GetNumAvailableBytes(void *_p) { + CMatchFinderMt *p = (CMatchFinderMt *)_p; if (p->btBufPos != p->btBufPosLimit) return p->btNumAvailBytes; return MatchFinderMt_GetNextBlock_Bt(p); @@ -1243,8 +1246,9 @@ static UInt32 * MixMatches4(CMatchFinderMt *p, UInt32 matchMinPos, UInt32 *d) } -static UInt32 * MatchFinderMt2_GetMatches(CMatchFinderMt *p, UInt32 *d) +static UInt32 * MatchFinderMt2_GetMatches(void *_p, UInt32 *d) { + CMatchFinderMt *p = (CMatchFinderMt *)_p; const UInt32 *bt = p->btBufPos; const UInt32 len = *bt++; const UInt32 *btLim = bt + len; @@ -1267,8 +1271,9 @@ static UInt32 * MatchFinderMt2_GetMatches(CMatchFinderMt *p, UInt32 *d) -static UInt32 * MatchFinderMt_GetMatches(CMatchFinderMt *p, UInt32 *d) +static UInt32 * MatchFinderMt_GetMatches(void *_p, UInt32 *d) { + CMatchFinderMt *p = (CMatchFinderMt *)_p; const UInt32 *bt = p->btBufPos; UInt32 len = *bt++; const UInt32 avail = p->btNumAvailBytes - 1; @@ -1315,14 +1320,16 @@ static UInt32 * MatchFinderMt_GetMatches(CMatchFinderMt *p, UInt32 *d) #define SKIP_HEADER_MT(n) SKIP_HEADER2_MT if (p->btNumAvailBytes-- >= (n)) { const Byte *cur = p->pointerToCurPos; UInt32 *hash = p->hash; #define SKIP_FOOTER_MT } INCREASE_LZ_POS p->btBufPos += (size_t)*p->btBufPos + 1; } while (--num != 0); -static void MatchFinderMt0_Skip(CMatchFinderMt *p, UInt32 num) +static void MatchFinderMt0_Skip(void *_p, UInt32 num) { + CMatchFinderMt *p = (CMatchFinderMt *)_p; SKIP_HEADER2_MT { p->btNumAvailBytes--; SKIP_FOOTER_MT } -static void MatchFinderMt2_Skip(CMatchFinderMt *p, UInt32 num) +static void MatchFinderMt2_Skip(void *_p, UInt32 num) { + CMatchFinderMt *p = (CMatchFinderMt *)_p; SKIP_HEADER_MT(2) UInt32 h2; MT_HASH2_CALC @@ -1330,8 +1337,9 @@ static void MatchFinderMt2_Skip(CMatchFinderMt *p, UInt32 num) SKIP_FOOTER_MT } -static void MatchFinderMt3_Skip(CMatchFinderMt *p, UInt32 num) +static void MatchFinderMt3_Skip(void *_p, UInt32 num) { + CMatchFinderMt *p = (CMatchFinderMt *)_p; SKIP_HEADER_MT(3) UInt32 h2, h3; MT_HASH3_CALC @@ -1361,39 +1369,39 @@ static void MatchFinderMt4_Skip(CMatchFinderMt *p, UInt32 num) void MatchFinderMt_CreateVTable(CMatchFinderMt *p, IMatchFinder2 *vTable) { - vTable->Init = (Mf_Init_Func)MatchFinderMt_Init; - vTable->GetNumAvailableBytes = (Mf_GetNumAvailableBytes_Func)MatchFinderMt_GetNumAvailableBytes; - vTable->GetPointerToCurrentPos = (Mf_GetPointerToCurrentPos_Func)MatchFinderMt_GetPointerToCurrentPos; - vTable->GetMatches = (Mf_GetMatches_Func)MatchFinderMt_GetMatches; + vTable->Init = MatchFinderMt_Init; + vTable->GetNumAvailableBytes = MatchFinderMt_GetNumAvailableBytes; + vTable->GetPointerToCurrentPos = MatchFinderMt_GetPointerToCurrentPos; + vTable->GetMatches = MatchFinderMt_GetMatches; switch (MF(p)->numHashBytes) { case 2: p->GetHeadsFunc = GetHeads2; - p->MixMatchesFunc = (Mf_Mix_Matches)NULL; - vTable->Skip = (Mf_Skip_Func)MatchFinderMt0_Skip; - vTable->GetMatches = (Mf_GetMatches_Func)MatchFinderMt2_GetMatches; + p->MixMatchesFunc = NULL; + vTable->Skip = MatchFinderMt0_Skip; + vTable->GetMatches = MatchFinderMt2_GetMatches; break; case 3: p->GetHeadsFunc = MF(p)->bigHash ? GetHeads3b : GetHeads3; - p->MixMatchesFunc = (Mf_Mix_Matches)MixMatches2; - vTable->Skip = (Mf_Skip_Func)MatchFinderMt2_Skip; + p->MixMatchesFunc = MixMatches2; + vTable->Skip = MatchFinderMt2_Skip; break; case 4: p->GetHeadsFunc = MF(p)->bigHash ? GetHeads4b : GetHeads4; // it's fast inline version of GetMatches() - // vTable->GetMatches = (Mf_GetMatches_Func)MatchFinderMt_GetMatches_Bt4; + // vTable->GetMatches = MatchFinderMt_GetMatches_Bt4; - p->MixMatchesFunc = (Mf_Mix_Matches)MixMatches3; - vTable->Skip = (Mf_Skip_Func)MatchFinderMt3_Skip; + p->MixMatchesFunc = MixMatches3; + vTable->Skip = MatchFinderMt3_Skip; break; default: p->GetHeadsFunc = MF(p)->bigHash ? GetHeads5b : GetHeads5; - p->MixMatchesFunc = (Mf_Mix_Matches)MixMatches4; + p->MixMatchesFunc = MixMatches4; vTable->Skip = - (Mf_Skip_Func)MatchFinderMt3_Skip; - // (Mf_Skip_Func)MatchFinderMt4_Skip; + MatchFinderMt3_Skip; + // MatchFinderMt4_Skip; break; } } diff --git a/C/LzFindMt.h b/C/LzFindMt.h index db5923e..fcb479d 100644 --- a/C/LzFindMt.h +++ b/C/LzFindMt.h @@ -1,5 +1,5 @@ /* LzFindMt.h -- multithreaded Match finder for LZ algorithms -2023-03-05 : Igor Pavlov : Public domain */ +2024-01-22 : Igor Pavlov : Public domain */ #ifndef ZIP7_INC_LZ_FIND_MT_H #define ZIP7_INC_LZ_FIND_MT_H @@ -31,7 +31,10 @@ typedef struct // UInt32 numBlocks_Sent; } CMtSync; -typedef UInt32 * (*Mf_Mix_Matches)(void *p, UInt32 matchMinPos, UInt32 *distances); + +struct CMatchFinderMt_; + +typedef UInt32 * (*Mf_Mix_Matches)(struct CMatchFinderMt_ *p, UInt32 matchMinPos, UInt32 *distances); /* kMtCacheLineDummy must be >= size_of_CPU_cache_line */ #define kMtCacheLineDummy 128 @@ -39,7 +42,7 @@ typedef UInt32 * (*Mf_Mix_Matches)(void *p, UInt32 matchMinPos, UInt32 *distance typedef void (*Mf_GetHeads)(const Byte *buffer, UInt32 pos, UInt32 *hash, UInt32 hashMask, UInt32 *heads, UInt32 numHeads, const UInt32 *crc); -typedef struct +typedef struct CMatchFinderMt_ { /* LZ */ const Byte *pointerToCurPos; diff --git a/C/Lzma2Dec.c b/C/Lzma2Dec.c index 388cbc7..8bf54e4 100644 --- a/C/Lzma2Dec.c +++ b/C/Lzma2Dec.c @@ -1,5 +1,5 @@ /* Lzma2Dec.c -- LZMA2 Decoder -2023-03-03 : Igor Pavlov : Public domain */ +2024-03-01 : Igor Pavlov : Public domain */ /* #define SHOW_DEBUG_INFO */ @@ -157,8 +157,10 @@ static unsigned Lzma2Dec_UpdateState(CLzma2Dec *p, Byte b) p->decoder.prop.lp = (Byte)lp; return LZMA2_STATE_DATA; } + + default: + return LZMA2_STATE_ERROR; } - return LZMA2_STATE_ERROR; } static void LzmaDec_UpdateWithUncompressed(CLzmaDec *p, const Byte *src, SizeT size) diff --git a/C/LzmaEnc.c b/C/LzmaEnc.c index 6d13cac..37b2787 100644 --- a/C/LzmaEnc.c +++ b/C/LzmaEnc.c @@ -1,5 +1,5 @@ /* LzmaEnc.c -- LZMA Encoder -2023-04-13: Igor Pavlov : Public domain */ +2024-01-24: Igor Pavlov : Public domain */ #include "Precomp.h" @@ -195,11 +195,11 @@ unsigned GetPosSlot1(UInt32 pos); unsigned GetPosSlot1(UInt32 pos) { unsigned res; - BSR2_RET(pos, res); + BSR2_RET(pos, res) return res; } -#define GetPosSlot2(pos, res) { BSR2_RET(pos, res); } -#define GetPosSlot(pos, res) { if (pos < 2) res = pos; else BSR2_RET(pos, res); } +#define GetPosSlot2(pos, res) { BSR2_RET(pos, res) } +#define GetPosSlot(pos, res) { if (pos < 2) res = pos; else BSR2_RET(pos, res) } #else // ! LZMA_LOG_BSR @@ -512,7 +512,7 @@ struct CLzmaEnc COPY_ARR(d, s, posEncoders) \ (d)->lenProbs = (s)->lenProbs; \ (d)->repLenProbs = (s)->repLenProbs; \ - memcpy((d)->litProbs, (s)->litProbs, ((UInt32)0x300 << (p)->lclp) * sizeof(CLzmaProb)); + memcpy((d)->litProbs, (s)->litProbs, ((size_t)0x300 * sizeof(CLzmaProb)) << (p)->lclp); void LzmaEnc_SaveState(CLzmaEncHandle p) { @@ -1040,14 +1040,14 @@ Z7_NO_INLINE static void Z7_FASTCALL LenPriceEnc_UpdateTables( UInt32 price = b; do { - unsigned bit = sym & 1; + const unsigned bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[sym], bit); } while (sym >= 2); { - unsigned prob = probs[(size_t)i + (1 << (kLenNumHighBits - 1))]; + const unsigned prob = probs[(size_t)i + (1 << (kLenNumHighBits - 1))]; prices[(size_t)i * 2 ] = price + GET_PRICEa_0(prob); prices[(size_t)i * 2 + 1] = price + GET_PRICEa_1(prob); } @@ -1056,7 +1056,7 @@ Z7_NO_INLINE static void Z7_FASTCALL LenPriceEnc_UpdateTables( { unsigned posState; - size_t num = (p->tableSize - kLenNumLowSymbols * 2) * sizeof(p->prices[0][0]); + const size_t num = (p->tableSize - kLenNumLowSymbols * 2) * sizeof(p->prices[0][0]); for (posState = 1; posState < numPosStates; posState++) memcpy(p->prices[posState] + kLenNumLowSymbols * 2, p->prices[0] + kLenNumLowSymbols * 2, num); } @@ -2696,12 +2696,12 @@ static SRes LzmaEnc_Alloc(CLzmaEnc *p, UInt32 keepWindowSize, ISzAllocPtr alloc, #endif { - unsigned lclp = p->lc + p->lp; + const unsigned lclp = p->lc + p->lp; if (!p->litProbs || !p->saveState.litProbs || p->lclp != lclp) { LzmaEnc_FreeLits(p, alloc); - p->litProbs = (CLzmaProb *)ISzAlloc_Alloc(alloc, ((UInt32)0x300 << lclp) * sizeof(CLzmaProb)); - p->saveState.litProbs = (CLzmaProb *)ISzAlloc_Alloc(alloc, ((UInt32)0x300 << lclp) * sizeof(CLzmaProb)); + p->litProbs = (CLzmaProb *)ISzAlloc_Alloc(alloc, ((size_t)0x300 * sizeof(CLzmaProb)) << lclp); + p->saveState.litProbs = (CLzmaProb *)ISzAlloc_Alloc(alloc, ((size_t)0x300 * sizeof(CLzmaProb)) << lclp); if (!p->litProbs || !p->saveState.litProbs) { LzmaEnc_FreeLits(p, alloc); @@ -2802,8 +2802,8 @@ static void LzmaEnc_Init(CLzmaEnc *p) } { - UInt32 num = (UInt32)0x300 << (p->lp + p->lc); - UInt32 k; + const size_t num = (size_t)0x300 << (p->lp + p->lc); + size_t k; CLzmaProb *probs = p->litProbs; for (k = 0; k < num; k++) probs[k] = kProbInitValue; diff --git a/C/MtCoder.c b/C/MtCoder.c index 6f58abb..03959b6 100644 --- a/C/MtCoder.c +++ b/C/MtCoder.c @@ -1,5 +1,5 @@ /* MtCoder.c -- Multi-thread Coder -2023-04-13 : Igor Pavlov : Public domain */ +2023-09-07 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -430,7 +430,7 @@ SRes MtCoder_Code(CMtCoder *p) SRes res = SZ_OK; if (numThreads > MTCODER_THREADS_MAX) - numThreads = MTCODER_THREADS_MAX; + numThreads = MTCODER_THREADS_MAX; numBlocksMax = MTCODER_GET_NUM_BLOCKS_FROM_THREADS(numThreads); if (p->blockSize < ((UInt32)1 << 26)) numBlocksMax++; @@ -438,7 +438,7 @@ SRes MtCoder_Code(CMtCoder *p) if (p->blockSize < ((UInt32)1 << 22)) numBlocksMax++; if (numBlocksMax > MTCODER_BLOCKS_MAX) - numBlocksMax = MTCODER_BLOCKS_MAX; + numBlocksMax = MTCODER_BLOCKS_MAX; if (p->blockSize != p->allocatedBufsSize) { @@ -469,7 +469,7 @@ SRes MtCoder_Code(CMtCoder *p) { RINOK_THREAD(AutoResetEvent_OptCreate_And_Reset(&p->readEvent)) - RINOK_THREAD(Semaphore_OptCreateInit(&p->blocksSemaphore, numBlocksMax, numBlocksMax)) + RINOK_THREAD(Semaphore_OptCreateInit(&p->blocksSemaphore, (UInt32)numBlocksMax, (UInt32)numBlocksMax)) } for (i = 0; i < MTCODER_BLOCKS_MAX - 1; i++) diff --git a/C/MtDec.c b/C/MtDec.c index 7820699..96274b6 100644 --- a/C/MtDec.c +++ b/C/MtDec.c @@ -1,5 +1,5 @@ /* MtDec.c -- Multi-thread Decoder -2023-04-02 : Igor Pavlov : Public domain */ +2024-02-20 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -809,6 +809,16 @@ static WRes MtDec_ThreadFunc2(CMtDecThread *t) #endif +typedef + #ifdef _WIN32 + UINT_PTR + #elif 1 + uintptr_t + #else + ptrdiff_t + #endif + MY_uintptr_t; + static THREAD_FUNC_DECL MtDec_ThreadFunc1(void *pp) { WRes res; @@ -821,7 +831,7 @@ static THREAD_FUNC_DECL MtDec_ThreadFunc1(void *pp) res = MtDec_ThreadFunc2(t); p = t->mtDec; if (res == 0) - return (THREAD_FUNC_RET_TYPE)(UINT_PTR)p->exitThreadWRes; + return (THREAD_FUNC_RET_TYPE)(MY_uintptr_t)p->exitThreadWRes; { // it's unexpected situation for some threading function error if (p->exitThreadWRes == 0) @@ -832,7 +842,7 @@ static THREAD_FUNC_DECL MtDec_ThreadFunc1(void *pp) Event_Set(&p->threads[0].canWrite); MtProgress_SetError(&p->mtProgress, MY_SRes_HRESULT_FROM_WRes(res)); } - return (THREAD_FUNC_RET_TYPE)(UINT_PTR)res; + return (THREAD_FUNC_RET_TYPE)(MY_uintptr_t)res; } static Z7_NO_INLINE THREAD_FUNC_DECL MtDec_ThreadFunc(void *pp) @@ -1072,7 +1082,7 @@ SRes MtDec_Code(CMtDec *p) if (wres == 0) { wres = Event_Set(&nextThread->canWrite); if (wres == 0) { wres = Event_Set(&nextThread->canRead); if (wres == 0) { THREAD_FUNC_RET_TYPE res = MtDec_ThreadFunc(nextThread); - wres = (WRes)(UINT_PTR)res; + wres = (WRes)(MY_uintptr_t)res; if (wres != 0) { p->needContinue = False; diff --git a/C/Ppmd7.c b/C/Ppmd7.c index 6e1307e..efcc5d8 100644 --- a/C/Ppmd7.c +++ b/C/Ppmd7.c @@ -1,5 +1,5 @@ /* Ppmd7.c -- PPMdH codec -2023-04-02 : Igor Pavlov : Public domain +2023-09-07 : Igor Pavlov : Public domain This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ #include "Precomp.h" @@ -302,8 +302,17 @@ static void *Ppmd7_AllocUnits(CPpmd7 *p, unsigned indx) #define MEM_12_CPY(dest, src, num) \ - { UInt32 *d = (UInt32 *)dest; const UInt32 *z = (const UInt32 *)src; UInt32 n = num; \ - do { d[0] = z[0]; d[1] = z[1]; d[2] = z[2]; z += 3; d += 3; } while (--n); } + { UInt32 *d = (UInt32 *)(dest); \ + const UInt32 *z = (const UInt32 *)(src); \ + unsigned n = (num); \ + do { \ + d[0] = z[0]; \ + d[1] = z[1]; \ + d[2] = z[2]; \ + z += 3; \ + d += 3; \ + } while (--n); \ + } /* @@ -711,8 +720,8 @@ void Ppmd7_UpdateModel(CPpmd7 *p) if ((ns1 & 1) == 0) { /* Expand for one UNIT */ - unsigned oldNU = ns1 >> 1; - unsigned i = U2I(oldNU); + const unsigned oldNU = ns1 >> 1; + const unsigned i = U2I(oldNU); if (i != U2I((size_t)oldNU + 1)) { void *ptr = Ppmd7_AllocUnits(p, i + 1); @@ -731,7 +740,7 @@ void Ppmd7_UpdateModel(CPpmd7 *p) sum = c->Union2.SummFreq; /* max increase of Escape_Freq is 3 here. total increase of Union2.SummFreq for all symbols is less than 256 here */ - sum += (UInt32)(2 * ns1 < ns) + 2 * ((unsigned)(4 * ns1 <= ns) & (sum <= 8 * ns1)); + sum += (UInt32)(unsigned)((2 * ns1 < ns) + 2 * ((unsigned)(4 * ns1 <= ns) & (sum <= 8 * ns1))); /* original PPMdH uses 16-bit variable for (sum) here. But (sum < 0x9000). So we don't truncate (sum) to 16-bit */ // sum = (UInt16)sum; @@ -761,7 +770,7 @@ void Ppmd7_UpdateModel(CPpmd7 *p) // (max(s->freq) == 120), when we convert from 1-symbol into 2-symbol context s->Freq = (Byte)freq; // max(InitEsc = PPMD7_kExpEscape[*]) is 25. So the max(escapeFreq) is 26 here - sum = freq + p->InitEsc + (ns > 3); + sum = (UInt32)(freq + p->InitEsc + (ns > 3)); } } @@ -933,10 +942,10 @@ CPpmd_See *Ppmd7_MakeEscFreq(CPpmd7 *p, unsigned numMasked, UInt32 *escFreq) p->HiBitsFlag; { // if (see->Summ) field is larger than 16-bit, we need only low 16 bits of Summ - unsigned summ = (UInt16)see->Summ; // & 0xFFFF - unsigned r = (summ >> see->Shift); + const unsigned summ = (UInt16)see->Summ; // & 0xFFFF + const unsigned r = (summ >> see->Shift); see->Summ = (UInt16)(summ - r); - *escFreq = r + (r == 0); + *escFreq = (UInt32)(r + (r == 0)); } } else @@ -981,9 +990,9 @@ void Ppmd7_Update1_0(CPpmd7 *p) CPpmd_State *s = p->FoundState; CPpmd7_Context *mc = p->MinContext; unsigned freq = s->Freq; - unsigned summFreq = mc->Union2.SummFreq; + const unsigned summFreq = mc->Union2.SummFreq; p->PrevSuccess = (2 * freq > summFreq); - p->RunLength += (int)p->PrevSuccess; + p->RunLength += (Int32)p->PrevSuccess; mc->Union2.SummFreq = (UInt16)(summFreq + 4); freq += 4; s->Freq = (Byte)freq; diff --git a/C/Ppmd7Dec.c b/C/Ppmd7Dec.c index 8323828..081ab89 100644 --- a/C/Ppmd7Dec.c +++ b/C/Ppmd7Dec.c @@ -1,5 +1,5 @@ /* Ppmd7Dec.c -- Ppmd7z (PPMdH with 7z Range Coder) Decoder -2023-04-02 : Igor Pavlov : Public domain +2023-09-07 : Igor Pavlov : Public domain This code is based on: PPMd var.H (2001): Dmitry Shkarin : Public domain */ @@ -58,7 +58,7 @@ static void Ppmd7z_RD_Decode(CPpmd7 *p, UInt32 start, UInt32 size) #define SUCCESSOR(p) Ppmd_GET_SUCCESSOR(p) void Ppmd7_UpdateModel(CPpmd7 *p); -#define MASK(sym) ((unsigned char *)charMask)[sym] +#define MASK(sym) ((Byte *)charMask)[sym] // Z7_FORCE_INLINE // static int Ppmd7z_DecodeSymbol(CPpmd7 *p) @@ -120,8 +120,8 @@ int Ppmd7z_DecodeSymbol(CPpmd7 *p) MASK(s->Symbol) = 0; do { - unsigned sym0 = s2[0].Symbol; - unsigned sym1 = s2[1].Symbol; + const unsigned sym0 = s2[0].Symbol; + const unsigned sym1 = s2[1].Symbol; s2 += 2; MASK(sym0) = 0; MASK(sym1) = 0; @@ -209,17 +209,17 @@ int Ppmd7z_DecodeSymbol(CPpmd7 *p) unsigned num2 = num / 2; num &= 1; - hiCnt = (s->Freq & (unsigned)(MASK(s->Symbol))) & (0 - (UInt32)num); + hiCnt = (s->Freq & (UInt32)(MASK(s->Symbol))) & (0 - (UInt32)num); s += num; p->MinContext = mc; do { - unsigned sym0 = s[0].Symbol; - unsigned sym1 = s[1].Symbol; + const unsigned sym0 = s[0].Symbol; + const unsigned sym1 = s[1].Symbol; s += 2; - hiCnt += (s[-2].Freq & (unsigned)(MASK(sym0))); - hiCnt += (s[-1].Freq & (unsigned)(MASK(sym1))); + hiCnt += (s[-2].Freq & (UInt32)(MASK(sym0))); + hiCnt += (s[-1].Freq & (UInt32)(MASK(sym1))); } while (--num2); } @@ -238,13 +238,13 @@ int Ppmd7z_DecodeSymbol(CPpmd7 *p) s = Ppmd7_GetStats(p, p->MinContext); hiCnt = count; - // count -= s->Freq & (unsigned)(MASK(s->Symbol)); + // count -= s->Freq & (UInt32)(MASK(s->Symbol)); // if ((Int32)count >= 0) { for (;;) { - count -= s->Freq & (unsigned)(MASK((s)->Symbol)); s++; if ((Int32)count < 0) break; - // count -= s->Freq & (unsigned)(MASK((s)->Symbol)); s++; if ((Int32)count < 0) break; + count -= s->Freq & (UInt32)(MASK((s)->Symbol)); s++; if ((Int32)count < 0) break; + // count -= s->Freq & (UInt32)(MASK((s)->Symbol)); s++; if ((Int32)count < 0) break; } } s--; diff --git a/C/Ppmd7Enc.c b/C/Ppmd7Enc.c index 41106ba..49cbbe6 100644 --- a/C/Ppmd7Enc.c +++ b/C/Ppmd7Enc.c @@ -1,5 +1,5 @@ /* Ppmd7Enc.c -- Ppmd7z (PPMdH with 7z Range Coder) Encoder -2023-04-02 : Igor Pavlov : Public domain +2023-09-07 : Igor Pavlov : Public domain This code is based on: PPMd var.H (2001): Dmitry Shkarin : Public domain */ @@ -82,7 +82,7 @@ void Ppmd7z_Flush_RangeEnc(CPpmd7 *p) void Ppmd7_UpdateModel(CPpmd7 *p); -#define MASK(sym) ((unsigned char *)charMask)[sym] +#define MASK(sym) ((Byte *)charMask)[sym] Z7_FORCE_INLINE static @@ -139,8 +139,8 @@ void Ppmd7z_EncodeSymbol(CPpmd7 *p, int symbol) MASK(s->Symbol) = 0; do { - unsigned sym0 = s2[0].Symbol; - unsigned sym1 = s2[1].Symbol; + const unsigned sym0 = s2[0].Symbol; + const unsigned sym1 = s2[1].Symbol; s2 += 2; MASK(sym0) = 0; MASK(sym1) = 0; @@ -265,16 +265,15 @@ void Ppmd7z_EncodeSymbol(CPpmd7 *p, int symbol) if (num2 != 0) { s += i; - for (;;) + do { - unsigned sym0 = s[0].Symbol; - unsigned sym1 = s[1].Symbol; + const unsigned sym0 = s[0].Symbol; + const unsigned sym1 = s[1].Symbol; s += 2; sum += (s[-2].Freq & (unsigned)(MASK(sym0))); sum += (s[-1].Freq & (unsigned)(MASK(sym1))); - if (--num2 == 0) - break; } + while (--num2); } diff --git a/C/Ppmd7aDec.c b/C/Ppmd7aDec.c index 55e164e..ef86dde 100644 --- a/C/Ppmd7aDec.c +++ b/C/Ppmd7aDec.c @@ -1,5 +1,5 @@ /* Ppmd7aDec.c -- PPMd7a (PPMdH) Decoder -2023-04-02 : Igor Pavlov : Public domain +2023-09-07 : Igor Pavlov : Public domain This code is based on: PPMd var.H (2001): Dmitry Shkarin : Public domain Carryless rangecoder (1999): Dmitry Subbotin : Public domain */ @@ -58,7 +58,7 @@ typedef CPpmd7_Context * CTX_PTR; #define SUCCESSOR(p) Ppmd_GET_SUCCESSOR(p) void Ppmd7_UpdateModel(CPpmd7 *p); -#define MASK(sym) ((unsigned char *)charMask)[sym] +#define MASK(sym) ((Byte *)charMask)[sym] int Ppmd7a_DecodeSymbol(CPpmd7 *p) @@ -120,8 +120,8 @@ int Ppmd7a_DecodeSymbol(CPpmd7 *p) MASK(s->Symbol) = 0; do { - unsigned sym0 = s2[0].Symbol; - unsigned sym1 = s2[1].Symbol; + const unsigned sym0 = s2[0].Symbol; + const unsigned sym1 = s2[1].Symbol; s2 += 2; MASK(sym0) = 0; MASK(sym1) = 0; @@ -209,17 +209,17 @@ int Ppmd7a_DecodeSymbol(CPpmd7 *p) unsigned num2 = num / 2; num &= 1; - hiCnt = (s->Freq & (unsigned)(MASK(s->Symbol))) & (0 - (UInt32)num); + hiCnt = (s->Freq & (UInt32)(MASK(s->Symbol))) & (0 - (UInt32)num); s += num; p->MinContext = mc; do { - unsigned sym0 = s[0].Symbol; - unsigned sym1 = s[1].Symbol; + const unsigned sym0 = s[0].Symbol; + const unsigned sym1 = s[1].Symbol; s += 2; - hiCnt += (s[-2].Freq & (unsigned)(MASK(sym0))); - hiCnt += (s[-1].Freq & (unsigned)(MASK(sym1))); + hiCnt += (s[-2].Freq & (UInt32)(MASK(sym0))); + hiCnt += (s[-1].Freq & (UInt32)(MASK(sym1))); } while (--num2); } @@ -238,13 +238,13 @@ int Ppmd7a_DecodeSymbol(CPpmd7 *p) s = Ppmd7_GetStats(p, p->MinContext); hiCnt = count; - // count -= s->Freq & (unsigned)(MASK(s->Symbol)); + // count -= s->Freq & (UInt32)(MASK(s->Symbol)); // if ((Int32)count >= 0) { for (;;) { - count -= s->Freq & (unsigned)(MASK((s)->Symbol)); s++; if ((Int32)count < 0) break; - // count -= s->Freq & (unsigned)(MASK((s)->Symbol)); s++; if ((Int32)count < 0) break; + count -= s->Freq & (UInt32)(MASK((s)->Symbol)); s++; if ((Int32)count < 0) break; + // count -= s->Freq & (UInt32)(MASK((s)->Symbol)); s++; if ((Int32)count < 0) break; } } s--; diff --git a/C/Ppmd8.c b/C/Ppmd8.c index 28abf27..c6bdd86 100644 --- a/C/Ppmd8.c +++ b/C/Ppmd8.c @@ -1,5 +1,5 @@ /* Ppmd8.c -- PPMdI codec -2023-04-02 : Igor Pavlov : Public domain +2023-09-07 : Igor Pavlov : Public domain This code is based on PPMd var.I (2002): Dmitry Shkarin : Public domain */ #include "Precomp.h" @@ -302,8 +302,17 @@ static void *Ppmd8_AllocUnits(CPpmd8 *p, unsigned indx) #define MEM_12_CPY(dest, src, num) \ - { UInt32 *d = (UInt32 *)dest; const UInt32 *z = (const UInt32 *)src; UInt32 n = num; \ - do { d[0] = z[0]; d[1] = z[1]; d[2] = z[2]; z += 3; d += 3; } while (--n); } + { UInt32 *d = (UInt32 *)(dest); \ + const UInt32 *z = (const UInt32 *)(src); \ + unsigned n = (num); \ + do { \ + d[0] = z[0]; \ + d[1] = z[1]; \ + d[2] = z[2]; \ + z += 3; \ + d += 3; \ + } while (--n); \ + } @@ -1215,8 +1224,8 @@ void Ppmd8_UpdateModel(CPpmd8 *p) if ((ns1 & 1) != 0) { /* Expand for one UNIT */ - unsigned oldNU = (ns1 + 1) >> 1; - unsigned i = U2I(oldNU); + const unsigned oldNU = (ns1 + 1) >> 1; + const unsigned i = U2I(oldNU); if (i != U2I((size_t)oldNU + 1)) { void *ptr = Ppmd8_AllocUnits(p, i + 1); @@ -1235,7 +1244,7 @@ void Ppmd8_UpdateModel(CPpmd8 *p) sum = c->Union2.SummFreq; /* max increase of Escape_Freq is 1 here. an average increase is 1/3 per symbol */ - sum += (3 * ns1 + 1 < ns); + sum += (UInt32)(unsigned)(3 * ns1 + 1 < ns); /* original PPMdH uses 16-bit variable for (sum) here. But (sum < ???). Do we need to truncate (sum) to 16-bit */ // sum = (UInt16)sum; @@ -1265,7 +1274,7 @@ void Ppmd8_UpdateModel(CPpmd8 *p) s->Freq = (Byte)freq; - sum = freq + p->InitEsc + (ns > 2); // Ppmd8 (> 2) + sum = (UInt32)(freq + p->InitEsc + (ns > 2)); // Ppmd8 (> 2) } } @@ -1437,10 +1446,10 @@ CPpmd_See *Ppmd8_MakeEscFreq(CPpmd8 *p, unsigned numMasked1, UInt32 *escFreq) { // if (see->Summ) field is larger than 16-bit, we need only low 16 bits of Summ - unsigned summ = (UInt16)see->Summ; // & 0xFFFF - unsigned r = (summ >> see->Shift); + const unsigned summ = (UInt16)see->Summ; // & 0xFFFF + const unsigned r = (summ >> see->Shift); see->Summ = (UInt16)(summ - r); - *escFreq = r + (r == 0); + *escFreq = (UInt32)(r + (r == 0)); } } else @@ -1485,9 +1494,9 @@ void Ppmd8_Update1_0(CPpmd8 *p) CPpmd_State *s = p->FoundState; CPpmd8_Context *mc = p->MinContext; unsigned freq = s->Freq; - unsigned summFreq = mc->Union2.SummFreq; + const unsigned summFreq = mc->Union2.SummFreq; p->PrevSuccess = (2 * freq >= summFreq); // Ppmd8 (>=) - p->RunLength += (int)p->PrevSuccess; + p->RunLength += (Int32)p->PrevSuccess; mc->Union2.SummFreq = (UInt16)(summFreq + 4); freq += 4; s->Freq = (Byte)freq; diff --git a/C/Ppmd8Dec.c b/C/Ppmd8Dec.c index 72d3626..ff91167 100644 --- a/C/Ppmd8Dec.c +++ b/C/Ppmd8Dec.c @@ -1,5 +1,5 @@ /* Ppmd8Dec.c -- Ppmd8 (PPMdI) Decoder -2023-04-02 : Igor Pavlov : Public domain +2023-09-07 : Igor Pavlov : Public domain This code is based on: PPMd var.I (2002): Dmitry Shkarin : Public domain Carryless rangecoder (1999): Dmitry Subbotin : Public domain */ @@ -58,7 +58,7 @@ static void Ppmd8_RD_Decode(CPpmd8 *p, UInt32 start, UInt32 size) #define SUCCESSOR(p) Ppmd_GET_SUCCESSOR(p) void Ppmd8_UpdateModel(CPpmd8 *p); -#define MASK(sym) ((unsigned char *)charMask)[sym] +#define MASK(sym) ((Byte *)charMask)[sym] int Ppmd8_DecodeSymbol(CPpmd8 *p) @@ -120,8 +120,8 @@ int Ppmd8_DecodeSymbol(CPpmd8 *p) MASK(s->Symbol) = 0; do { - unsigned sym0 = s2[0].Symbol; - unsigned sym1 = s2[1].Symbol; + const unsigned sym0 = s2[0].Symbol; + const unsigned sym1 = s2[1].Symbol; s2 += 2; MASK(sym0) = 0; MASK(sym1) = 0; @@ -209,17 +209,17 @@ int Ppmd8_DecodeSymbol(CPpmd8 *p) unsigned num2 = num / 2; num &= 1; - hiCnt = (s->Freq & (unsigned)(MASK(s->Symbol))) & (0 - (UInt32)num); + hiCnt = (s->Freq & (UInt32)(MASK(s->Symbol))) & (0 - (UInt32)num); s += num; p->MinContext = mc; do { - unsigned sym0 = s[0].Symbol; - unsigned sym1 = s[1].Symbol; + const unsigned sym0 = s[0].Symbol; + const unsigned sym1 = s[1].Symbol; s += 2; - hiCnt += (s[-2].Freq & (unsigned)(MASK(sym0))); - hiCnt += (s[-1].Freq & (unsigned)(MASK(sym1))); + hiCnt += (s[-2].Freq & (UInt32)(MASK(sym0))); + hiCnt += (s[-1].Freq & (UInt32)(MASK(sym1))); } while (--num2); } @@ -243,8 +243,8 @@ int Ppmd8_DecodeSymbol(CPpmd8 *p) { for (;;) { - count -= s->Freq & (unsigned)(MASK((s)->Symbol)); s++; if ((Int32)count < 0) break; - // count -= s->Freq & (unsigned)(MASK((s)->Symbol)); s++; if ((Int32)count < 0) break; + count -= s->Freq & (UInt32)(MASK((s)->Symbol)); s++; if ((Int32)count < 0) break; + // count -= s->Freq & (UInt32)(MASK((s)->Symbol)); s++; if ((Int32)count < 0) break; } } s--; diff --git a/C/Ppmd8Enc.c b/C/Ppmd8Enc.c index 9e29ef7..b0e34c4 100644 --- a/C/Ppmd8Enc.c +++ b/C/Ppmd8Enc.c @@ -1,5 +1,5 @@ /* Ppmd8Enc.c -- Ppmd8 (PPMdI) Encoder -2023-04-02 : Igor Pavlov : Public domain +2023-09-07 : Igor Pavlov : Public domain This code is based on: PPMd var.I (2002): Dmitry Shkarin : Public domain Carryless rangecoder (1999): Dmitry Subbotin : Public domain */ @@ -82,7 +82,7 @@ static void Ppmd8_RangeEnc_Encode(CPpmd8 *p, UInt32 start, UInt32 size, UInt32 t void Ppmd8_UpdateModel(CPpmd8 *p); -#define MASK(sym) ((unsigned char *)charMask)[sym] +#define MASK(sym) ((Byte *)charMask)[sym] // Z7_FORCE_INLINE // static @@ -139,8 +139,8 @@ void Ppmd8_EncodeSymbol(CPpmd8 *p, int symbol) MASK(s->Symbol) = 0; do { - unsigned sym0 = s2[0].Symbol; - unsigned sym1 = s2[1].Symbol; + const unsigned sym0 = s2[0].Symbol; + const unsigned sym1 = s2[1].Symbol; s2 += 2; MASK(sym0) = 0; MASK(sym1) = 0; @@ -265,16 +265,15 @@ void Ppmd8_EncodeSymbol(CPpmd8 *p, int symbol) if (num2 != 0) { s += i; - for (;;) + do { - unsigned sym0 = s[0].Symbol; - unsigned sym1 = s[1].Symbol; + const unsigned sym0 = s[0].Symbol; + const unsigned sym1 = s[1].Symbol; s += 2; sum += (s[-2].Freq & (unsigned)(MASK(sym0))); sum += (s[-1].Freq & (unsigned)(MASK(sym1))); - if (--num2 == 0) - break; } + while (--num2); } PPMD8_CORRECT_SUM_RANGE(p, sum) diff --git a/C/Precomp.h b/C/Precomp.h index 69afb2f..7747fdd 100644 --- a/C/Precomp.h +++ b/C/Precomp.h @@ -1,10 +1,127 @@ -/* Precomp.h -- StdAfx -2023-04-02 : Igor Pavlov : Public domain */ +/* Precomp.h -- precompilation file +2024-01-25 : Igor Pavlov : Public domain */ #ifndef ZIP7_INC_PRECOMP_H #define ZIP7_INC_PRECOMP_H +/* + this file must be included before another *.h files and before . + this file is included from the following files: + C\*.c + C\Util\*\Precomp.h <- C\Util\*\*.c + CPP\Common\Common.h <- *\StdAfx.h <- *\*.cpp + + this file can set the following macros: + Z7_LARGE_PAGES 1 + Z7_LONG_PATH 1 + Z7_WIN32_WINNT_MIN 0x0500 (or higher) : we require at least win2000+ for 7-Zip + _WIN32_WINNT 0x0500 (or higher) + WINVER _WIN32_WINNT + UNICODE 1 + _UNICODE 1 +*/ + #include "Compiler.h" -/* #include "7zTypes.h" */ + +#ifdef _MSC_VER +// #pragma warning(disable : 4206) // nonstandard extension used : translation unit is empty +#if _MSC_VER >= 1912 +// #pragma warning(disable : 5039) // pointer or reference to potentially throwing function passed to 'extern "C"' function under - EHc.Undefined behavior may occur if this function throws an exception. +#endif +#endif + +/* +// for debug: +#define UNICODE 1 +#define _UNICODE 1 +#define _WIN32_WINNT 0x0500 // win2000 +#ifndef WINVER + #define WINVER _WIN32_WINNT +#endif +*/ + +#ifdef _WIN32 +/* + this "Precomp.h" file must be included before , + if we want to define _WIN32_WINNT before . +*/ + +#ifndef Z7_LARGE_PAGES +#ifndef Z7_NO_LARGE_PAGES +#define Z7_LARGE_PAGES 1 +#endif +#endif + +#ifndef Z7_LONG_PATH +#ifndef Z7_NO_LONG_PATH +#define Z7_LONG_PATH 1 +#endif +#endif + +#ifndef Z7_DEVICE_FILE +#ifndef Z7_NO_DEVICE_FILE +// #define Z7_DEVICE_FILE 1 +#endif +#endif + +// we don't change macros if included after +#ifndef _WINDOWS_ + +#ifndef Z7_WIN32_WINNT_MIN + #if defined(_M_ARM64) || defined(__aarch64__) + // #define Z7_WIN32_WINNT_MIN 0x0a00 // win10 + #define Z7_WIN32_WINNT_MIN 0x0600 // vista + #elif defined(_M_ARM) && defined(_M_ARMT) && defined(_M_ARM_NT) + // #define Z7_WIN32_WINNT_MIN 0x0602 // win8 + #define Z7_WIN32_WINNT_MIN 0x0600 // vista + #elif defined(_M_X64) || defined(_M_AMD64) || defined(__x86_64__) || defined(_M_IA64) + #define Z7_WIN32_WINNT_MIN 0x0503 // win2003 + // #elif defined(_M_IX86) || defined(__i386__) + // #define Z7_WIN32_WINNT_MIN 0x0500 // win2000 + #else // x86 and another(old) systems + #define Z7_WIN32_WINNT_MIN 0x0500 // win2000 + // #define Z7_WIN32_WINNT_MIN 0x0502 // win2003 // for debug + #endif +#endif // Z7_WIN32_WINNT_MIN + + +#ifndef Z7_DO_NOT_DEFINE_WIN32_WINNT +#ifdef _WIN32_WINNT + // #error Stop_Compiling_Bad_WIN32_WINNT +#else + #ifndef Z7_NO_DEFINE_WIN32_WINNT +Z7_DIAGNOSTIC_IGNORE_BEGIN_RESERVED_MACRO_IDENTIFIER + #define _WIN32_WINNT Z7_WIN32_WINNT_MIN +Z7_DIAGNOSTIC_IGNORE_END_RESERVED_MACRO_IDENTIFIER + #endif +#endif // _WIN32_WINNT + +#ifndef WINVER + #define WINVER _WIN32_WINNT +#endif +#endif // Z7_DO_NOT_DEFINE_WIN32_WINNT + + +#ifndef _MBCS +#ifndef Z7_NO_UNICODE +// UNICODE and _UNICODE are used by and by 7-zip code. + +#ifndef UNICODE +#define UNICODE 1 +#endif + +#ifndef _UNICODE +Z7_DIAGNOSTIC_IGNORE_BEGIN_RESERVED_MACRO_IDENTIFIER +#define _UNICODE 1 +Z7_DIAGNOSTIC_IGNORE_END_RESERVED_MACRO_IDENTIFIER +#endif + +#endif // Z7_NO_UNICODE +#endif // _MBCS +#endif // _WINDOWS_ + +// #include "7zWindows.h" + +#endif // _WIN32 #endif diff --git a/C/Sha1.c b/C/Sha1.c index fd6c018..4c92892 100644 --- a/C/Sha1.c +++ b/C/Sha1.c @@ -1,5 +1,5 @@ /* Sha1.c -- SHA-1 Hash -2023-04-02 : Igor Pavlov : Public domain +2024-03-01 : Igor Pavlov : Public domain This code is based on public domain code of Steve Reid from Wei Dai's Crypto++ library. */ #include "Precomp.h" @@ -15,35 +15,35 @@ This code is based on public domain code of Steve Reid from Wei Dai's Crypto++ l #endif #ifdef MY_CPU_X86_OR_AMD64 - #ifdef _MSC_VER - #if _MSC_VER >= 1200 + #if defined(Z7_LLVM_CLANG_VERSION) && (Z7_LLVM_CLANG_VERSION >= 30800) \ + || defined(Z7_APPLE_CLANG_VERSION) && (Z7_APPLE_CLANG_VERSION >= 50100) \ + || defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 40900) \ + || defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1600) \ + || defined(_MSC_VER) && (_MSC_VER >= 1200) #define Z7_COMPILER_SHA1_SUPPORTED - #endif - #elif defined(__clang__) - #if (__clang_major__ >= 8) // fix that check - #define Z7_COMPILER_SHA1_SUPPORTED - #endif - #elif defined(__GNUC__) - #if (__GNUC__ >= 8) // fix that check - #define Z7_COMPILER_SHA1_SUPPORTED - #endif - #elif defined(__INTEL_COMPILER) - #if (__INTEL_COMPILER >= 1800) // fix that check - #define Z7_COMPILER_SHA1_SUPPORTED - #endif #endif -#elif defined(MY_CPU_ARM_OR_ARM64) - #ifdef _MSC_VER - #if _MSC_VER >= 1910 && _MSC_VER >= 1929 && _MSC_FULL_VER >= 192930037 +#elif defined(MY_CPU_ARM_OR_ARM64) && defined(MY_CPU_LE) \ + && (!defined(Z7_MSC_VER_ORIGINAL) || (_MSC_VER >= 1929) && (_MSC_FULL_VER >= 192930037)) + #if defined(__ARM_FEATURE_SHA2) \ + || defined(__ARM_FEATURE_CRYPTO) + #define Z7_COMPILER_SHA1_SUPPORTED + #else + #if defined(MY_CPU_ARM64) \ + || defined(__ARM_ARCH) && (__ARM_ARCH >= 4) \ + || defined(Z7_MSC_VER_ORIGINAL) + #if defined(__ARM_FP) && \ + ( defined(Z7_CLANG_VERSION) && (Z7_CLANG_VERSION >= 30800) \ + || defined(__GNUC__) && (__GNUC__ >= 6) \ + ) \ + || defined(Z7_MSC_VER_ORIGINAL) && (_MSC_VER >= 1910) + #if defined(MY_CPU_ARM64) \ + || !defined(Z7_CLANG_VERSION) \ + || defined(__ARM_NEON) && \ + (Z7_CLANG_VERSION < 170000 || \ + Z7_CLANG_VERSION > 170001) #define Z7_COMPILER_SHA1_SUPPORTED #endif - #elif defined(__clang__) - #if (__clang_major__ >= 8) // fix that check - #define Z7_COMPILER_SHA1_SUPPORTED #endif - #elif defined(__GNUC__) - #if (__GNUC__ >= 6) // fix that check - #define Z7_COMPILER_SHA1_SUPPORTED #endif #endif #endif @@ -436,7 +436,7 @@ void Sha1Prepare(void) #endif { // printf("\n========== HW SHA1 ======== \n"); - #if defined(MY_CPU_ARM_OR_ARM64) && defined(_MSC_VER) + #if 0 && defined(MY_CPU_ARM_OR_ARM64) && defined(_MSC_VER) /* there was bug in MSVC compiler for ARM64 -O2 before version VS2019 16.10 (19.29.30037). It generated incorrect SHA-1 code. 21.03 : we test sha1-hardware code at runtime initialization */ diff --git a/C/Sha1Opt.c b/C/Sha1Opt.c index 27796aa..4e835f1 100644 --- a/C/Sha1Opt.c +++ b/C/Sha1Opt.c @@ -1,5 +1,5 @@ /* Sha1Opt.c -- SHA-1 optimized code for SHA-1 hardware instructions -2023-04-02 : Igor Pavlov : Public domain */ +2024-03-01 : Igor Pavlov : Public domain */ #include "Precomp.h" #include "Compiler.h" @@ -11,6 +11,8 @@ #endif #endif +// #define Z7_USE_HW_SHA_STUB // for debug + #ifdef MY_CPU_X86_OR_AMD64 #if defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1600) // fix that check #define USE_HW_SHA @@ -32,9 +34,14 @@ #endif #if (_MSC_VER >= USE_VER_MIN) #define USE_HW_SHA + #else + #define Z7_USE_HW_SHA_STUB #endif #endif // #endif // MY_CPU_X86_OR_AMD64 +#ifndef USE_HW_SHA + // #define Z7_USE_HW_SHA_STUB // for debug +#endif #ifdef USE_HW_SHA @@ -202,46 +209,124 @@ void Z7_FASTCALL Sha1_UpdateBlocks_HW(UInt32 state[5], const Byte *data, size_t #endif // USE_HW_SHA -#elif defined(MY_CPU_ARM_OR_ARM64) - - #if defined(__clang__) - #if (__clang_major__ >= 8) // fix that check +#elif defined(MY_CPU_ARM_OR_ARM64) && defined(MY_CPU_LE) \ + && (!defined(Z7_MSC_VER_ORIGINAL) || (_MSC_VER >= 1929) && (_MSC_FULL_VER >= 192930037)) + #if defined(__ARM_FEATURE_SHA2) \ + || defined(__ARM_FEATURE_CRYPTO) + #define USE_HW_SHA + #else + #if defined(MY_CPU_ARM64) \ + || defined(__ARM_ARCH) && (__ARM_ARCH >= 4) \ + || defined(Z7_MSC_VER_ORIGINAL) + #if defined(__ARM_FP) && \ + ( defined(Z7_CLANG_VERSION) && (Z7_CLANG_VERSION >= 30800) \ + || defined(__GNUC__) && (__GNUC__ >= 6) \ + ) \ + || defined(Z7_MSC_VER_ORIGINAL) && (_MSC_VER >= 1910) + #if defined(MY_CPU_ARM64) \ + || !defined(Z7_CLANG_VERSION) \ + || defined(__ARM_NEON) && \ + (Z7_CLANG_VERSION < 170000 || \ + Z7_CLANG_VERSION > 170001) #define USE_HW_SHA #endif - #elif defined(__GNUC__) - #if (__GNUC__ >= 6) // fix that check - #define USE_HW_SHA #endif - #elif defined(_MSC_VER) - #if _MSC_VER >= 1910 - #define USE_HW_SHA #endif #endif #ifdef USE_HW_SHA // #pragma message("=== Sha1 HW === ") +// __ARM_FEATURE_CRYPTO macro is deprecated in favor of the finer grained feature macro __ARM_FEATURE_SHA2 #if defined(__clang__) || defined(__GNUC__) +#if !defined(__ARM_FEATURE_SHA2) && \ + !defined(__ARM_FEATURE_CRYPTO) #ifdef MY_CPU_ARM64 +#if defined(__clang__) + #define ATTRIB_SHA __attribute__((__target__("crypto"))) +#else #define ATTRIB_SHA __attribute__((__target__("+crypto"))) +#endif #else +#if defined(__clang__) && (__clang_major__ >= 1) + #define ATTRIB_SHA __attribute__((__target__("armv8-a,sha2"))) +#else #define ATTRIB_SHA __attribute__((__target__("fpu=crypto-neon-fp-armv8"))) +#endif #endif +#endif #else // _MSC_VER // for arm32 #define _ARM_USE_NEW_NEON_INTRINSICS #endif -#if defined(_MSC_VER) && defined(MY_CPU_ARM64) + + + + +#if defined(Z7_MSC_VER_ORIGINAL) && defined(MY_CPU_ARM64) #include #else + + + + + + + + + +#if defined(__clang__) && __clang_major__ < 16 +#if !defined(__ARM_FEATURE_SHA2) && \ + !defined(__ARM_FEATURE_CRYPTO) +// #pragma message("=== we set __ARM_FEATURE_CRYPTO 1 === ") + Z7_DIAGNOSTIC_IGNORE_BEGIN_RESERVED_MACRO_IDENTIFIER + #define Z7_ARM_FEATURE_CRYPTO_WAS_SET 1 +// #if defined(__clang__) && __clang_major__ < 13 + #define __ARM_FEATURE_CRYPTO 1 +// #else + #define __ARM_FEATURE_SHA2 1 +// #endif + Z7_DIAGNOSTIC_IGNORE_END_RESERVED_MACRO_IDENTIFIER +#endif +#endif // clang + +#if defined(__clang__) + +#if defined(__ARM_ARCH) && __ARM_ARCH < 8 + Z7_DIAGNOSTIC_IGNORE_BEGIN_RESERVED_MACRO_IDENTIFIER +// #pragma message("#define __ARM_ARCH 8") + #undef __ARM_ARCH + #define __ARM_ARCH 8 + Z7_DIAGNOSTIC_IGNORE_END_RESERVED_MACRO_IDENTIFIER +#endif + +#endif // clang + #include + +#if defined(Z7_ARM_FEATURE_CRYPTO_WAS_SET) && \ + defined(__ARM_FEATURE_CRYPTO) && \ + defined(__ARM_FEATURE_SHA2) +Z7_DIAGNOSTIC_IGNORE_BEGIN_RESERVED_MACRO_IDENTIFIER + #undef __ARM_FEATURE_CRYPTO + #undef __ARM_FEATURE_SHA2 + #undef Z7_ARM_FEATURE_CRYPTO_WAS_SET +Z7_DIAGNOSTIC_IGNORE_END_RESERVED_MACRO_IDENTIFIER +// #pragma message("=== we undefine __ARM_FEATURE_CRYPTO === ") #endif +#endif // Z7_MSC_VER_ORIGINAL + typedef uint32x4_t v128; // typedef __n128 v128; // MSVC +// the bug in clang 3.8.1: +// __builtin_neon_vgetq_lane_i32((int8x16_t)__s0, __p1); +#if defined(__clang__) && (__clang_major__ <= 9) +#pragma GCC diagnostic ignored "-Wvector-conversion" +#endif #ifdef MY_CPU_BE #define MY_rev32_for_LE(x) @@ -256,11 +341,11 @@ typedef uint32x4_t v128; m = LOAD_128((data + (k) * 16)); \ MY_rev32_for_LE(m); \ -#define SU0(dest, src2, src3) dest = vsha1su0q_u32(dest, src2, src3); -#define SU1(dest, src) dest = vsha1su1q_u32(dest, src); -#define C(e) abcd = vsha1cq_u32(abcd, e, t); -#define P(e) abcd = vsha1pq_u32(abcd, e, t); -#define M(e) abcd = vsha1mq_u32(abcd, e, t); +#define SU0(dest, src2, src3) dest = vsha1su0q_u32(dest, src2, src3) +#define SU1(dest, src) dest = vsha1su1q_u32(dest, src) +#define C(e) abcd = vsha1cq_u32(abcd, e, t) +#define P(e) abcd = vsha1pq_u32(abcd, e, t) +#define M(e) abcd = vsha1mq_u32(abcd, e, t) #define H(e) e = vsha1h_u32(vgetq_lane_u32(abcd, 0)) #define T(m, c) t = vaddq_u32(m, c) @@ -337,16 +422,17 @@ void Z7_FASTCALL Sha1_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_t #endif // MY_CPU_ARM_OR_ARM64 -#ifndef USE_HW_SHA - +#if !defined(USE_HW_SHA) && defined(Z7_USE_HW_SHA_STUB) // #error Stop_Compiling_UNSUPPORTED_SHA // #include -// #include "Sha1.h" -void Z7_FASTCALL Sha1_UpdateBlocks(UInt32 state[5], const Byte *data, size_t numBlocks); -#pragma message("Sha1 HW-SW stub was used") +// #include "Sha1.h" +// #if defined(_MSC_VER) +#pragma message("Sha1 HW-SW stub was used") +// #endif +void Z7_FASTCALL Sha1_UpdateBlocks (UInt32 state[5], const Byte *data, size_t numBlocks); void Z7_FASTCALL Sha1_UpdateBlocks_HW(UInt32 state[5], const Byte *data, size_t numBlocks); void Z7_FASTCALL Sha1_UpdateBlocks_HW(UInt32 state[5], const Byte *data, size_t numBlocks) { @@ -359,7 +445,6 @@ void Z7_FASTCALL Sha1_UpdateBlocks_HW(UInt32 state[5], const Byte *data, size_t return; */ } - #endif #undef SU0 @@ -384,3 +469,4 @@ void Z7_FASTCALL Sha1_UpdateBlocks_HW(UInt32 state[5], const Byte *data, size_t #undef USE_HW_SHA #undef ATTRIB_SHA #undef USE_VER_MIN +#undef Z7_USE_HW_SHA_STUB diff --git a/C/Sha256.c b/C/Sha256.c index 018cf6f..14d3be9 100644 --- a/C/Sha256.c +++ b/C/Sha256.c @@ -1,5 +1,5 @@ /* Sha256.c -- SHA-256 Hash -2023-04-02 : Igor Pavlov : Public domain +2024-03-01 : Igor Pavlov : Public domain This code is based on public domain code from Wei Dai's Crypto++ library. */ #include "Precomp.h" @@ -15,35 +15,35 @@ This code is based on public domain code from Wei Dai's Crypto++ library. */ #endif #ifdef MY_CPU_X86_OR_AMD64 - #ifdef _MSC_VER - #if _MSC_VER >= 1200 + #if defined(Z7_LLVM_CLANG_VERSION) && (Z7_LLVM_CLANG_VERSION >= 30800) \ + || defined(Z7_APPLE_CLANG_VERSION) && (Z7_APPLE_CLANG_VERSION >= 50100) \ + || defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 40900) \ + || defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1600) \ + || defined(_MSC_VER) && (_MSC_VER >= 1200) #define Z7_COMPILER_SHA256_SUPPORTED - #endif - #elif defined(__clang__) - #if (__clang_major__ >= 8) // fix that check - #define Z7_COMPILER_SHA256_SUPPORTED - #endif - #elif defined(__GNUC__) - #if (__GNUC__ >= 8) // fix that check - #define Z7_COMPILER_SHA256_SUPPORTED - #endif - #elif defined(__INTEL_COMPILER) - #if (__INTEL_COMPILER >= 1800) // fix that check - #define Z7_COMPILER_SHA256_SUPPORTED - #endif #endif -#elif defined(MY_CPU_ARM_OR_ARM64) - #ifdef _MSC_VER - #if _MSC_VER >= 1910 +#elif defined(MY_CPU_ARM_OR_ARM64) && defined(MY_CPU_LE) + + #if defined(__ARM_FEATURE_SHA2) \ + || defined(__ARM_FEATURE_CRYPTO) + #define Z7_COMPILER_SHA256_SUPPORTED + #else + #if defined(MY_CPU_ARM64) \ + || defined(__ARM_ARCH) && (__ARM_ARCH >= 4) \ + || defined(Z7_MSC_VER_ORIGINAL) + #if defined(__ARM_FP) && \ + ( defined(Z7_CLANG_VERSION) && (Z7_CLANG_VERSION >= 30800) \ + || defined(__GNUC__) && (__GNUC__ >= 6) \ + ) \ + || defined(Z7_MSC_VER_ORIGINAL) && (_MSC_VER >= 1910) + #if defined(MY_CPU_ARM64) \ + || !defined(Z7_CLANG_VERSION) \ + || defined(__ARM_NEON) && \ + (Z7_CLANG_VERSION < 170000 || \ + Z7_CLANG_VERSION > 170001) #define Z7_COMPILER_SHA256_SUPPORTED #endif - #elif defined(__clang__) - #if (__clang_major__ >= 8) // fix that check - #define Z7_COMPILER_SHA256_SUPPORTED #endif - #elif defined(__GNUC__) - #if (__GNUC__ >= 6) // fix that check - #define Z7_COMPILER_SHA256_SUPPORTED #endif #endif #endif @@ -224,8 +224,6 @@ void Sha256_Init(CSha256 *p) #endif -void Z7_FASTCALL Sha256_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_t numBlocks); - // static extern MY_ALIGN(64) const UInt32 SHA256_K_ARRAY[64]; diff --git a/C/Sha256Opt.c b/C/Sha256Opt.c index e4465e3..eb38166 100644 --- a/C/Sha256Opt.c +++ b/C/Sha256Opt.c @@ -1,5 +1,5 @@ /* Sha256Opt.c -- SHA-256 optimized code for SHA-256 hardware instructions -2023-04-02 : Igor Pavlov : Public domain */ +2024-03-01 : Igor Pavlov : Public domain */ #include "Precomp.h" #include "Compiler.h" @@ -11,6 +11,8 @@ #endif #endif +// #define Z7_USE_HW_SHA_STUB // for debug + #ifdef MY_CPU_X86_OR_AMD64 #if defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 1600) // fix that check #define USE_HW_SHA @@ -32,9 +34,14 @@ #endif #if (_MSC_VER >= USE_VER_MIN) #define USE_HW_SHA + #else + #define Z7_USE_HW_SHA_STUB #endif #endif // #endif // MY_CPU_X86_OR_AMD64 +#ifndef USE_HW_SHA + // #define Z7_USE_HW_SHA_STUB // for debug +#endif #ifdef USE_HW_SHA @@ -202,19 +209,28 @@ void Z7_FASTCALL Sha256_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_ #endif // USE_HW_SHA -#elif defined(MY_CPU_ARM_OR_ARM64) - - #if defined(__clang__) - #if (__clang_major__ >= 8) // fix that check +#elif defined(MY_CPU_ARM_OR_ARM64) && defined(MY_CPU_LE) + + #if defined(__ARM_FEATURE_SHA2) \ + || defined(__ARM_FEATURE_CRYPTO) + #define USE_HW_SHA + #else + #if defined(MY_CPU_ARM64) \ + || defined(__ARM_ARCH) && (__ARM_ARCH >= 4) \ + || defined(Z7_MSC_VER_ORIGINAL) + #if defined(__ARM_FP) && \ + ( defined(Z7_CLANG_VERSION) && (Z7_CLANG_VERSION >= 30800) \ + || defined(__GNUC__) && (__GNUC__ >= 6) \ + ) \ + || defined(Z7_MSC_VER_ORIGINAL) && (_MSC_VER >= 1910) + #if defined(MY_CPU_ARM64) \ + || !defined(Z7_CLANG_VERSION) \ + || defined(__ARM_NEON) && \ + (Z7_CLANG_VERSION < 170000 || \ + Z7_CLANG_VERSION > 170001) #define USE_HW_SHA #endif - #elif defined(__GNUC__) - #if (__GNUC__ >= 6) // fix that check - #define USE_HW_SHA #endif - #elif defined(_MSC_VER) - #if _MSC_VER >= 1910 - #define USE_HW_SHA #endif #endif @@ -222,24 +238,88 @@ void Z7_FASTCALL Sha256_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_ // #pragma message("=== Sha256 HW === ") + #if defined(__clang__) || defined(__GNUC__) +#if !defined(__ARM_FEATURE_SHA2) && \ + !defined(__ARM_FEATURE_CRYPTO) #ifdef MY_CPU_ARM64 +#if defined(__clang__) + #define ATTRIB_SHA __attribute__((__target__("crypto"))) +#else #define ATTRIB_SHA __attribute__((__target__("+crypto"))) +#endif #else +#if defined(__clang__) && (__clang_major__ >= 1) + #define ATTRIB_SHA __attribute__((__target__("armv8-a,sha2"))) +#else #define ATTRIB_SHA __attribute__((__target__("fpu=crypto-neon-fp-armv8"))) +#endif #endif +#endif #else // _MSC_VER // for arm32 #define _ARM_USE_NEW_NEON_INTRINSICS #endif -#if defined(_MSC_VER) && defined(MY_CPU_ARM64) + + + + +#if defined(Z7_MSC_VER_ORIGINAL) && defined(MY_CPU_ARM64) #include #else + + + + + + + + + +#if defined(__clang__) && __clang_major__ < 16 +#if !defined(__ARM_FEATURE_SHA2) && \ + !defined(__ARM_FEATURE_CRYPTO) +// #pragma message("=== we set __ARM_FEATURE_CRYPTO 1 === ") + Z7_DIAGNOSTIC_IGNORE_BEGIN_RESERVED_MACRO_IDENTIFIER + #define Z7_ARM_FEATURE_CRYPTO_WAS_SET 1 +// #if defined(__clang__) && __clang_major__ < 13 + #define __ARM_FEATURE_CRYPTO 1 +// #else + #define __ARM_FEATURE_SHA2 1 +// #endif + Z7_DIAGNOSTIC_IGNORE_END_RESERVED_MACRO_IDENTIFIER +#endif +#endif // clang + +#if defined(__clang__) + +#if defined(__ARM_ARCH) && __ARM_ARCH < 8 + Z7_DIAGNOSTIC_IGNORE_BEGIN_RESERVED_MACRO_IDENTIFIER +// #pragma message("#define __ARM_ARCH 8") + #undef __ARM_ARCH + #define __ARM_ARCH 8 + Z7_DIAGNOSTIC_IGNORE_END_RESERVED_MACRO_IDENTIFIER +#endif + +#endif // clang + #include + +#if defined(Z7_ARM_FEATURE_CRYPTO_WAS_SET) && \ + defined(__ARM_FEATURE_CRYPTO) && \ + defined(__ARM_FEATURE_SHA2) +Z7_DIAGNOSTIC_IGNORE_BEGIN_RESERVED_MACRO_IDENTIFIER + #undef __ARM_FEATURE_CRYPTO + #undef __ARM_FEATURE_SHA2 + #undef Z7_ARM_FEATURE_CRYPTO_WAS_SET +Z7_DIAGNOSTIC_IGNORE_END_RESERVED_MACRO_IDENTIFIER +// #pragma message("=== we undefine __ARM_FEATURE_CRYPTO === ") #endif +#endif // Z7_MSC_VER_ORIGINAL + typedef uint32x4_t v128; // typedef __n128 v128; // MSVC @@ -316,10 +396,10 @@ void Z7_FASTCALL Sha256_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_ LOAD_SHUFFLE (m2, 2) LOAD_SHUFFLE (m3, 3) - R16 ( 0, NNN, NNN, SM1, NNN, SM1, SM2, SM1, SM2 ); - R16 ( 1, SM1, SM2, SM1, SM2, SM1, SM2, SM1, SM2 ); - R16 ( 2, SM1, SM2, SM1, SM2, SM1, SM2, SM1, SM2 ); - R16 ( 3, SM1, SM2, NNN, SM2, NNN, NNN, NNN, NNN ); + R16 ( 0, NNN, NNN, SM1, NNN, SM1, SM2, SM1, SM2 ) + R16 ( 1, SM1, SM2, SM1, SM2, SM1, SM2, SM1, SM2 ) + R16 ( 2, SM1, SM2, SM1, SM2, SM1, SM2, SM1, SM2 ) + R16 ( 3, SM1, SM2, NNN, SM2, NNN, NNN, NNN, NNN ) state0 = vaddq_u32(state0, state0_save); state1 = vaddq_u32(state1, state1_save); @@ -337,16 +417,17 @@ void Z7_FASTCALL Sha256_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_ #endif // MY_CPU_ARM_OR_ARM64 -#ifndef USE_HW_SHA - +#if !defined(USE_HW_SHA) && defined(Z7_USE_HW_SHA_STUB) // #error Stop_Compiling_UNSUPPORTED_SHA // #include - +// We can compile this file with another C compiler, +// or we can compile asm version. +// So we can generate real code instead of this stub function. // #include "Sha256.h" -void Z7_FASTCALL Sha256_UpdateBlocks(UInt32 state[8], const Byte *data, size_t numBlocks); - +// #if defined(_MSC_VER) #pragma message("Sha256 HW-SW stub was used") - +// #endif +void Z7_FASTCALL Sha256_UpdateBlocks (UInt32 state[8], const Byte *data, size_t numBlocks); void Z7_FASTCALL Sha256_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_t numBlocks); void Z7_FASTCALL Sha256_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_t numBlocks) { @@ -359,7 +440,6 @@ void Z7_FASTCALL Sha256_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_ return; */ } - #endif @@ -384,3 +464,4 @@ void Z7_FASTCALL Sha256_UpdateBlocks_HW(UInt32 state[8], const Byte *data, size_ #undef USE_HW_SHA #undef ATTRIB_SHA #undef USE_VER_MIN +#undef Z7_USE_HW_SHA_STUB diff --git a/C/SwapBytes.c b/C/SwapBytes.c index 7901bba..9290592 100644 --- a/C/SwapBytes.c +++ b/C/SwapBytes.c @@ -1,5 +1,5 @@ /* SwapBytes.c -- Byte Swap conversion filter -2023-04-07 : Igor Pavlov : Public domain */ +2024-03-01 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -305,11 +305,12 @@ ShufBytes_256(void *items8, const void *lim8, const void *mask128_ptr) msvc 19.30+ (VS2022): replaces _mm256_set_m128i(m,m) to vbroadcastf128(m) as we want */ // _mm256_broadcastsi128_si256(*mask128_ptr); - /* +#if defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION < 80000) #define MY_mm256_set_m128i(hi, lo) _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1) - MY_mm256_set_m128i - */ - _mm256_set_m128i( +#else + #define MY_mm256_set_m128i _mm256_set_m128i +#endif + MY_mm256_set_m128i( *(const __m128i *)mask128_ptr, *(const __m128i *)mask128_ptr); #endif @@ -330,32 +331,59 @@ ShufBytes_256(void *items8, const void *lim8, const void *mask128_ptr) // compile message "NEON intrinsics not available with the soft-float ABI" -#elif defined(MY_CPU_ARM_OR_ARM64) || \ - (defined(__ARM_ARCH) && (__ARM_ARCH >= 7)) -// #elif defined(MY_CPU_ARM64) +#elif defined(MY_CPU_ARM_OR_ARM64) \ + && defined(MY_CPU_LE) \ + && !defined(Z7_DISABLE_ARM_NEON) #if defined(__clang__) && (__clang_major__ >= 8) \ - || defined(__GNUC__) && (__GNUC__ >= 8) - #if (defined(__ARM_ARCH) && (__ARM_ARCH >= 7)) \ + || defined(__GNUC__) && (__GNUC__ >= 6) + #if defined(__ARM_FP) + #if (defined(__ARM_ARCH) && (__ARM_ARCH >= 4)) \ || defined(MY_CPU_ARM64) + #if defined(MY_CPU_ARM64) \ + || !defined(Z7_CLANG_VERSION) \ + || defined(__ARM_NEON) #define USE_SWAP_128 - #endif #ifdef MY_CPU_ARM64 // #define SWAP_ATTRIB_NEON __attribute__((__target__(""))) #else - // #define SWAP_ATTRIB_NEON __attribute__((__target__("fpu=crypto-neon-fp-armv8"))) - #endif +#if defined(Z7_CLANG_VERSION) + // #define SWAP_ATTRIB_NEON __attribute__((__target__("neon"))) +#else + // #pragma message("SWAP_ATTRIB_NEON __attribute__((__target__(fpu=neon))") + #define SWAP_ATTRIB_NEON __attribute__((__target__("fpu=neon"))) +#endif + #endif // MY_CPU_ARM64 + #endif // __ARM_NEON + #endif // __ARM_ARCH + #endif // __ARM_FP + #elif defined(_MSC_VER) #if (_MSC_VER >= 1910) #define USE_SWAP_128 #endif #endif - #if defined(_MSC_VER) && defined(MY_CPU_ARM64) + #ifdef USE_SWAP_128 + #if defined(Z7_MSC_VER_ORIGINAL) && defined(MY_CPU_ARM64) #include #else + +/* +#if !defined(__ARM_NEON) +#if defined(Z7_GCC_VERSION) && (__GNUC__ < 5) \ + || defined(Z7_GCC_VERSION) && (__GNUC__ == 5) && (Z7_GCC_VERSION < 90201) \ + || defined(Z7_GCC_VERSION) && (__GNUC__ == 5) && (Z7_GCC_VERSION < 100100) +Z7_DIAGNOSTIC_IGNORE_BEGIN_RESERVED_MACRO_IDENTIFIER +#pragma message("#define __ARM_NEON 1") +// #define __ARM_NEON 1 +Z7_DIAGNOSTIC_IGNORE_END_RESERVED_MACRO_IDENTIFIER +#endif +#endif +*/ #include #endif + #endif #ifndef USE_SWAP_128 #define FORCE_SWAP_MODE @@ -464,6 +492,13 @@ Z7_ATTRIB_NO_VECTOR \ void Z7_FASTCALL +#if defined(MY_CPU_ARM_OR_ARM64) +#if defined(__clang__) +#pragma GCC diagnostic ignored "-Wlanguage-extension-token" +#endif +#endif + + #ifdef MY_CPU_64BIT #if defined(MY_CPU_ARM64) \ diff --git a/C/Threads.c b/C/Threads.c index cf52bd3..464efec 100644 --- a/C/Threads.c +++ b/C/Threads.c @@ -1,5 +1,5 @@ /* Threads.c -- multithreading library -2023-03-04 : Igor Pavlov : Public domain */ +2024-03-28 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -195,20 +195,19 @@ WRes CriticalSection_Init(CCriticalSection *p) // ---------- POSIX ---------- -#ifndef __APPLE__ +#if defined(__linux__) && !defined(__APPLE__) && !defined(_AIX) && !defined(__ANDROID__) #ifndef Z7_AFFINITY_DISABLE // _GNU_SOURCE can be required for pthread_setaffinity_np() / CPU_ZERO / CPU_SET // clang < 3.6 : unknown warning group '-Wreserved-id-macro' // clang 3.6 - 12.01 : gives warning "macro name is a reserved identifier" // clang >= 13 : do not give warning #if !defined(_GNU_SOURCE) - #if defined(__clang__) && (__clang_major__ >= 4) && (__clang_major__ <= 12) - #pragma GCC diagnostic ignored "-Wreserved-id-macro" - #endif -#define _GNU_SOURCE +Z7_DIAGNOSTIC_IGNORE_BEGIN_RESERVED_MACRO_IDENTIFIER +// #define _GNU_SOURCE +Z7_DIAGNOSTIC_IGNORE_END_RESERVED_MACRO_IDENTIFIER #endif // !defined(_GNU_SOURCE) #endif // Z7_AFFINITY_DISABLE -#endif // __APPLE__ +#endif // __linux__ #include "Threads.h" @@ -244,8 +243,9 @@ WRes Thread_Create_With_CpuSet(CThread *p, THREAD_FUNC_TYPE func, LPVOID param, { if (cpuSet) { - #ifdef Z7_AFFINITY_SUPPORTED - + // pthread_attr_setaffinity_np() is not supported for MUSL compile. + // so we check for __GLIBC__ here +#if defined(Z7_AFFINITY_SUPPORTED) && defined( __GLIBC__) /* printf("\n affinity :"); unsigned i; @@ -267,7 +267,7 @@ WRes Thread_Create_With_CpuSet(CThread *p, THREAD_FUNC_TYPE func, LPVOID param, // ret2 = pthread_attr_setaffinity_np(&attr, sizeof(*cpuSet), cpuSet); // if (ret2) ret = ret2; - #endif +#endif } ret = pthread_create(&p->_tid, &attr, func, param); @@ -369,13 +369,20 @@ WRes AutoResetEvent_CreateNotSignaled(CAutoResetEvent *p) { return AutoResetEvent_Create(p, 0); } +#if defined(Z7_LLVM_CLANG_VERSION) && (__clang_major__ == 13) +// freebsd: +#pragma GCC diagnostic ignored "-Wthread-safety-analysis" +#endif + WRes Event_Set(CEvent *p) { RINOK(pthread_mutex_lock(&p->_mutex)) p->_state = True; - int res1 = pthread_cond_broadcast(&p->_cond); - int res2 = pthread_mutex_unlock(&p->_mutex); - return (res2 ? res2 : res1); + { + const int res1 = pthread_cond_broadcast(&p->_cond); + const int res2 = pthread_mutex_unlock(&p->_mutex); + return (res2 ? res2 : res1); + } } WRes Event_Reset(CEvent *p) @@ -408,8 +415,8 @@ WRes Event_Close(CEvent *p) return 0; p->_created = 0; { - int res1 = pthread_mutex_destroy(&p->_mutex); - int res2 = pthread_cond_destroy(&p->_cond); + const int res1 = pthread_mutex_destroy(&p->_mutex); + const int res2 = pthread_cond_destroy(&p->_cond); return (res1 ? res1 : res2); } } @@ -487,8 +494,8 @@ WRes Semaphore_Close(CSemaphore *p) return 0; p->_created = 0; { - int res1 = pthread_mutex_destroy(&p->_mutex); - int res2 = pthread_cond_destroy(&p->_cond); + const int res1 = pthread_mutex_destroy(&p->_mutex); + const int res2 = pthread_cond_destroy(&p->_cond); return (res1 ? res1 : res2); } } @@ -549,6 +556,18 @@ LONG InterlockedIncrement(LONG volatile *addend) #endif } +LONG InterlockedDecrement(LONG volatile *addend) +{ + // Print("InterlockedDecrement") + #ifdef USE_HACK_UNSAFE_ATOMIC + LONG val = *addend - 1; + *addend = val; + return val; + #else + return __sync_sub_and_fetch(addend, 1); + #endif +} + #endif // _WIN32 WRes AutoResetEvent_OptCreate_And_Reset(CAutoResetEvent *p) diff --git a/C/Threads.h b/C/Threads.h index 4028464..c1484a2 100644 --- a/C/Threads.h +++ b/C/Threads.h @@ -1,5 +1,5 @@ /* Threads.h -- multithreading library -2023-04-02 : Igor Pavlov : Public domain */ +2024-03-28 : Igor Pavlov : Public domain */ #ifndef ZIP7_INC_THREADS_H #define ZIP7_INC_THREADS_H @@ -9,12 +9,21 @@ #else +#include "Compiler.h" + +// #define Z7_AFFINITY_DISABLE #if defined(__linux__) #if !defined(__APPLE__) && !defined(_AIX) && !defined(__ANDROID__) #ifndef Z7_AFFINITY_DISABLE #define Z7_AFFINITY_SUPPORTED // #pragma message(" ==== Z7_AFFINITY_SUPPORTED") -// #define _GNU_SOURCE +#if !defined(_GNU_SOURCE) +// #pragma message(" ==== _GNU_SOURCE set") +// we need _GNU_SOURCE for cpu_set_t, if we compile for MUSL +Z7_DIAGNOSTIC_IGNORE_BEGIN_RESERVED_MACRO_IDENTIFIER +#define _GNU_SOURCE +Z7_DIAGNOSTIC_IGNORE_END_RESERVED_MACRO_IDENTIFIER +#endif #endif #endif #endif @@ -173,7 +182,7 @@ WRes CriticalSection_Init(CCriticalSection *p); #else // _WIN32 -typedef struct _CEvent +typedef struct { int _created; int _manual_reset; @@ -199,7 +208,7 @@ WRes Event_Wait(CEvent *p); WRes Event_Close(CEvent *p); -typedef struct _CSemaphore +typedef struct { int _created; UInt32 _count; @@ -219,7 +228,7 @@ WRes Semaphore_Wait(CSemaphore *p); WRes Semaphore_Close(CSemaphore *p); -typedef struct _CCriticalSection +typedef struct { pthread_mutex_t _mutex; } CCriticalSection; @@ -230,6 +239,7 @@ void CriticalSection_Enter(CCriticalSection *cs); void CriticalSection_Leave(CCriticalSection *cs); LONG InterlockedIncrement(LONG volatile *addend); +LONG InterlockedDecrement(LONG volatile *addend); #endif // _WIN32 diff --git a/C/Util/7z/7z.dsp b/C/Util/7z/7z.dsp index 11e1b03..474c660 100644 --- a/C/Util/7z/7z.dsp +++ b/C/Util/7z/7z.dsp @@ -42,7 +42,7 @@ RSC=rc.exe # PROP Ignore_Export_Lib 0 # PROP Target_Dir "" # ADD BASE CPP /nologo /W3 /GX /O2 /D "WIN32" /D "NDEBUG" /D "_CONSOLE" /D "_MBCS" /YX /FD /c -# ADD CPP /nologo /MD /W4 /WX /GX /O2 /D "NDEBUG" /D "WIN32" /D "_CONSOLE" /D "_UNICODE" /D "UNICODE" /D "Z7_PPMD_SUPPORT" /FAcs /Yu"Precomp.h" /FD /c +# ADD CPP /nologo /MD /W4 /WX /GX /O2 /D "NDEBUG" /D "WIN32" /D "_CONSOLE" /D "_UNICODE" /D "UNICODE" /D "Z7_PPMD_SUPPORT" /D "Z7_EXTRACT_ONLY" /FAcs /Yu"Precomp.h" /FD /c # ADD BASE RSC /l 0x419 /d "NDEBUG" # ADD RSC /l 0x419 /d "NDEBUG" BSC32=bscmake.exe @@ -67,7 +67,7 @@ LINK32=link.exe # PROP Ignore_Export_Lib 0 # PROP Target_Dir "" # ADD BASE CPP /nologo /W3 /Gm /GX /ZI /Od /D "WIN32" /D "_DEBUG" /D "_CONSOLE" /D "_MBCS" /YX /FD /GZ /c -# ADD CPP /nologo /W4 /WX /Gm /GX /ZI /Od /D "_DEBUG" /D "_SZ_ALLOC_DEBUG2" /D "_SZ_NO_INT_64_A" /D "WIN32" /D "_CONSOLE" /D "_UNICODE" /D "UNICODE" /D "Z7_PPMD_SUPPORT" /Yu"Precomp.h" /FD /GZ /c +# ADD CPP /nologo /W4 /WX /Gm /GX /ZI /Od /D "_DEBUG" /D "_SZ_ALLOC_DEBUG2" /D "_SZ_NO_INT_64_A" /D "WIN32" /D "_CONSOLE" /D "_UNICODE" /D "UNICODE" /D "Z7_PPMD_SUPPORT" /D "Z7_EXTRACT_ONLY" /Yu"Precomp.h" /FD /GZ /c # ADD BASE RSC /l 0x419 /d "_DEBUG" # ADD RSC /l 0x419 /d "_DEBUG" BSC32=bscmake.exe @@ -234,6 +234,10 @@ SOURCE=.\Precomp.c # End Source File # Begin Source File +SOURCE=..\..\Precomp.h +# End Source File +# Begin Source File + SOURCE=.\Precomp.h # End Source File # End Group diff --git a/C/Util/7z/7zMain.c b/C/Util/7z/7zMain.c index 547920a..6baf979 100644 --- a/C/Util/7z/7zMain.c +++ b/C/Util/7z/7zMain.c @@ -1,20 +1,11 @@ /* 7zMain.c - Test application for 7z Decoder -2023-04-04 : Igor Pavlov : Public domain */ +2024-02-28 : Igor Pavlov : Public domain */ #include "Precomp.h" #include #include -#include "../../CpuArch.h" - -#include "../../7z.h" -#include "../../7zAlloc.h" -#include "../../7zBuf.h" -#include "../../7zCrc.h" -#include "../../7zFile.h" -#include "../../7zVersion.h" - #ifndef USE_WINDOWS_FILE /* for mkdir */ #ifdef _WIN32 @@ -32,6 +23,15 @@ #endif #endif +#include "../../7zFile.h" +#include "../../7z.h" +#include "../../7zAlloc.h" +#include "../../7zBuf.h" +#include "../../7zCrc.h" +#include "../../7zVersion.h" + +#include "../../CpuArch.h" + #define kInputBufSize ((size_t)1 << 18) static const ISzAlloc g_Alloc = { SzAlloc, SzFree }; @@ -168,12 +168,12 @@ static SRes Utf16_To_Char(CBuf *buf, const UInt16 *s #endif ) { - unsigned len = 0; + size_t len = 0; for (len = 0; s[len] != 0; len++) {} #ifndef MY_USE_UTF8 { - const unsigned size = len * 3 + 100; + const size_t size = len * 3 + 100; if (!Buf_EnsureSize(buf, size)) return SZ_ERROR_MEM; { @@ -320,21 +320,20 @@ static void UIntToStr_2(char *s, unsigned value) // typedef long BOOL; typedef int BOOL; -typedef struct _FILETIME +typedef struct { DWORD dwLowDateTime; DWORD dwHighDateTime; } FILETIME; -static LONG TIME_GetBias() +static LONG TIME_GetBias(void) { const time_t utc = time(NULL); struct tm *ptm = localtime(&utc); const int localdaylight = ptm->tm_isdst; /* daylight for local timezone */ ptm = gmtime(&utc); ptm->tm_isdst = localdaylight; /* use local daylight, not that of Greenwich */ - const LONG bias = (int)(mktime(ptm) - utc); - return bias; + return (int)(mktime(ptm) - utc); } #define TICKS_PER_SEC 10000000 @@ -359,11 +358,11 @@ static BOOL WINAPI FileTimeToLocalFileTime(const FILETIME *fileTime, FILETIME *l static const UInt32 kNumTimeQuantumsInSecond = 10000000; static const UInt32 kFileTimeStartYear = 1601; static const UInt32 kUnixTimeStartYear = 1970; -static const UInt64 kUnixTimeOffset = - (UInt64)60 * 60 * 24 * (89 + 365 * (kUnixTimeStartYear - kFileTimeStartYear)); static Int64 Time_FileTimeToUnixTime64(const FILETIME *ft) { + const UInt64 kUnixTimeOffset = + (UInt64)60 * 60 * 24 * (89 + 365 * (kUnixTimeStartYear - kFileTimeStartYear)); const UInt64 winTime = GET_TIME_64(ft); return (Int64)(winTime / kNumTimeQuantumsInSecond) - (Int64)kUnixTimeOffset; } @@ -384,8 +383,10 @@ static void FILETIME_To_timespec(const FILETIME *ft, struct MY_ST_TIMESPEC *ts) if (sec2 == sec) { ts->tv_sec = sec2; - const UInt64 winTime = GET_TIME_64(ft); - ts->tv_nsec = (long)((winTime % 10000000) * 100); + { + const UInt64 winTime = GET_TIME_64(ft); + ts->tv_nsec = (long)((winTime % 10000000) * 100); + } return; } } @@ -429,7 +430,7 @@ static void ConvertFileTimeToString(const CNtfsFileTime *nTime, char *s) { unsigned year, mon, hour, min, sec; Byte ms[] = { 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 }; - unsigned t; + UInt32 t; UInt32 v; // UInt64 v64 = nt->Low | ((UInt64)nt->High << 32); UInt64 v64; @@ -461,7 +462,7 @@ static void ConvertFileTimeToString(const CNtfsFileTime *nTime, char *s) ms[1] = 29; for (mon = 0;; mon++) { - const unsigned d = ms[mon]; + const UInt32 d = ms[mon]; if (v < d) break; v -= d; diff --git a/C/Util/7z/Precomp.h b/C/Util/7z/Precomp.h index bc8fa21..13a41ef 100644 --- a/C/Util/7z/Precomp.h +++ b/C/Util/7z/Precomp.h @@ -1,14 +1,13 @@ -/* Precomp.h -- StdAfx -2023-03-04 : Igor Pavlov : Public domain */ +/* Precomp.h -- Precomp +2024-01-23 : Igor Pavlov : Public domain */ -#ifndef ZIP7_INC_PRECOMP_H -#define ZIP7_INC_PRECOMP_H +// #ifndef ZIP7_INC_PRECOMP_LOC_H +// #define ZIP7_INC_PRECOMP_LOC_H #if defined(_MSC_VER) && _MSC_VER >= 1800 #pragma warning(disable : 4464) // relative include path contains '..' #endif -#include "../../Compiler.h" -#include "../../7zTypes.h" +#include "../../Precomp.h" -#endif +// #endif diff --git a/C/Util/7z/makefile b/C/Util/7z/makefile index dfc560e..987f065 100644 --- a/C/Util/7z/makefile +++ b/C/Util/7z/makefile @@ -5,8 +5,6 @@ PROG = 7zDec.exe C_OBJS = \ $O\7zAlloc.obj \ $O\7zBuf.obj \ - $O\7zCrc.obj \ - $O\7zCrcOpt.obj \ $O\7zFile.obj \ $O\7zDec.obj \ $O\7zArcIn.obj \ @@ -25,10 +23,14 @@ C_OBJS = \ 7Z_OBJS = \ $O\7zMain.obj \ +!include "../../../CPP/7zip/Crc.mak" +!include "../../../CPP/7zip/LzmaDec.mak" + OBJS = \ $O\Precomp.obj \ $(7Z_OBJS) \ $(C_OBJS) \ + $(ASM_OBJS) \ !include "../../../CPP/Build.mak" @@ -38,3 +40,5 @@ $(C_OBJS): ../../$(*B).c $(CCOMPL_USE) $O\Precomp.obj: Precomp.c $(CCOMPL_PCH) + +!include "../../Asm_c.mak" diff --git a/C/Util/7zipInstall/7zipInstall.c b/C/Util/7zipInstall/7zipInstall.c index 7f5fd19..7d8e8c4 100644 --- a/C/Util/7zipInstall/7zipInstall.c +++ b/C/Util/7zipInstall/7zipInstall.c @@ -1,5 +1,5 @@ /* 7zipInstall.c - 7-Zip Installer -2023-04-04 : Igor Pavlov : Public domain */ +2024-04-05 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -11,6 +11,8 @@ #pragma warning(disable : 4201) // nonstandard extension used : nameless struct/union #endif +Z7_DIAGNOSTIC_IGNORE_CAST_FUNCTION + #ifdef Z7_OLD_WIN_SDK struct IShellView; #define SHFOLDERAPI EXTERN_C DECLSPEC_IMPORT HRESULT STDAPICALLTYPE @@ -41,16 +43,6 @@ typedef enum { // #pragma GCC diagnostic ignored "-Wcast-function-type" #endif -#if defined(__clang__) || defined(__GNUC__) -typedef void (*Z7_voidFunction)(void); -#define MY_CAST_FUNC (Z7_voidFunction) -#elif defined(_MSC_VER) && _MSC_VER > 1920 -#define MY_CAST_FUNC (void *) -// #pragma warning(disable : 4191) // 'type cast': unsafe conversion from 'FARPROC' to 'void (__cdecl *)()' -#else -#define MY_CAST_FUNC -#endif - #define LLL_(quote) L##quote #define LLL(quote) LLL_(quote) @@ -118,11 +110,13 @@ static LPCWSTR const k_Reg_Path32 = L"Path" #define k_Reg_WOW_Flag 0 #endif +#ifdef USE_7ZIP_32_DLL #ifdef _WIN64 #define k_Reg_WOW_Flag_32 KEY_WOW64_32KEY #else #define k_Reg_WOW_Flag_32 0 #endif +#endif #define k_7zip_CLSID L"{23170F69-40C1-278A-1000-000100020000}" @@ -219,11 +213,11 @@ static DWORD GetFileVersion(LPCWSTR s) return 0; } - my_GetFileVersionInfoSizeW = (Func_GetFileVersionInfoSizeW) MY_CAST_FUNC GetProcAddress(g_version_dll_hModule, + my_GetFileVersionInfoSizeW = (Func_GetFileVersionInfoSizeW) Z7_CAST_FUNC_C GetProcAddress(g_version_dll_hModule, "GetFileVersionInfoSizeW"); - my_GetFileVersionInfoW = (Func_GetFileVersionInfoW) MY_CAST_FUNC GetProcAddress(g_version_dll_hModule, + my_GetFileVersionInfoW = (Func_GetFileVersionInfoW) Z7_CAST_FUNC_C GetProcAddress(g_version_dll_hModule, "GetFileVersionInfoW"); - my_VerQueryValueW = (Func_VerQueryValueW) MY_CAST_FUNC GetProcAddress(g_version_dll_hModule, + my_VerQueryValueW = (Func_VerQueryValueW) Z7_CAST_FUNC_C GetProcAddress(g_version_dll_hModule, "VerQueryValueW"); if (!my_GetFileVersionInfoSizeW @@ -1102,7 +1096,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, { BOOL isWow64 = FALSE; const Func_IsWow64Process func_IsWow64Process = (Func_IsWow64Process) - MY_CAST_FUNC GetProcAddress(GetModuleHandleW(L"kernel32.dll"), + Z7_CAST_FUNC_C GetProcAddress(GetModuleHandleW(L"kernel32.dll"), "IsWow64Process"); if (func_IsWow64Process) @@ -1111,7 +1105,13 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, if (!isWow64) { if (!g_SilentMode) - PrintErrorMessage("This installation requires Windows " MY_CPU_NAME, NULL); + PrintErrorMessage("This installation requires Windows " + #ifdef MY_CPU_X86_OR_AMD64 + "x64" + #else + "64-bit" + #endif + , NULL); return 1; } } diff --git a/C/Util/7zipInstall/Precomp.h b/C/Util/7zipInstall/Precomp.h index bc8fa21..13a41ef 100644 --- a/C/Util/7zipInstall/Precomp.h +++ b/C/Util/7zipInstall/Precomp.h @@ -1,14 +1,13 @@ -/* Precomp.h -- StdAfx -2023-03-04 : Igor Pavlov : Public domain */ +/* Precomp.h -- Precomp +2024-01-23 : Igor Pavlov : Public domain */ -#ifndef ZIP7_INC_PRECOMP_H -#define ZIP7_INC_PRECOMP_H +// #ifndef ZIP7_INC_PRECOMP_LOC_H +// #define ZIP7_INC_PRECOMP_LOC_H #if defined(_MSC_VER) && _MSC_VER >= 1800 #pragma warning(disable : 4464) // relative include path contains '..' #endif -#include "../../Compiler.h" -#include "../../7zTypes.h" +#include "../../Precomp.h" -#endif +// #endif diff --git a/C/Util/7zipInstall/makefile b/C/Util/7zipInstall/makefile index 18e2783..424bd6c 100644 --- a/C/Util/7zipInstall/makefile +++ b/C/Util/7zipInstall/makefile @@ -19,9 +19,6 @@ C_OBJS = \ $O\7zAlloc.obj \ $O\7zArcIn.obj \ $O\7zBuf.obj \ - $O\7zBuf2.obj \ - $O\7zCrc.obj \ - $O\7zCrcOpt.obj \ $O\7zFile.obj \ $O\7zDec.obj \ $O\7zStream.obj \ @@ -34,11 +31,17 @@ C_OBJS = \ OBJS = \ $(MAIN_OBJS) \ $(C_OBJS) \ + $(ASM_OBJS) \ $O\resource.res +!include "../../../CPP/7zip/Crc.mak" +# !include "../../../CPP/7zip/LzmaDec.mak" + !include "../../../CPP/Build.mak" $(MAIN_OBJS): $(*B).c $(COMPL_O1) $(C_OBJS): ../../$(*B).c $(COMPL_O1) + +!include "../../Asm_c.mak" diff --git a/C/Util/7zipInstall/resource.rc b/C/Util/7zipInstall/resource.rc index df6474e..40ed580 100644 --- a/C/Util/7zipInstall/resource.rc +++ b/C/Util/7zipInstall/resource.rc @@ -1,5 +1,6 @@ -#include -#include +#include +// #include +// #include #include #define USE_COPYRIGHT_CR diff --git a/C/Util/7zipUninstall/7zipUninstall.c b/C/Util/7zipUninstall/7zipUninstall.c index 8bc18b3..e7051e2 100644 --- a/C/Util/7zipUninstall/7zipUninstall.c +++ b/C/Util/7zipUninstall/7zipUninstall.c @@ -1,10 +1,11 @@ /* 7zipUninstall.c - 7-Zip Uninstaller -2022-07-15 : Igor Pavlov : Public domain */ +2024-03-21 : Igor Pavlov : Public domain */ #include "Precomp.h" // #define SZ_ERROR_ABORT 100 +#include "../../7zTypes.h" #include "../../7zWindows.h" #if defined(_MSC_VER) && _MSC_VER < 1600 @@ -31,16 +32,7 @@ typedef enum { #include "resource.h" -#if (defined(__GNUC__) && (__GNUC__ >= 8)) || defined(__clang__) - // #pragma GCC diagnostic ignored "-Wcast-function-type" -#endif -#if defined(_MSC_VER) && _MSC_VER > 1920 -#define MY_CAST_FUNC (void *) -// #pragma warning(disable : 4191) // 'type cast': unsafe conversion from 'FARPROC' to 'void (__cdecl *)()' -#else -#define MY_CAST_FUNC -#endif #define LLL_(quote) L##quote @@ -101,11 +93,13 @@ static LPCWSTR const k_Reg_Path32 = L"Path" #define k_Reg_WOW_Flag 0 #endif +#ifdef USE_7ZIP_32_DLL #ifdef _WIN64 #define k_Reg_WOW_Flag_32 KEY_WOW64_32KEY #else #define k_Reg_WOW_Flag_32 0 #endif +#endif #define k_7zip_CLSID L"{23170F69-40C1-278A-1000-000100020000}" @@ -124,9 +118,19 @@ static HWND g_Path_HWND; static HWND g_InfoLine_HWND; static HWND g_Progress_HWND; -// WINADVAPI +// RegDeleteKeyExW is supported starting from win2003sp1/xp-pro-x64 +// Z7_WIN32_WINNT_MIN < 0x0600 // Vista +#if !defined(Z7_WIN32_WINNT_MIN) \ + || Z7_WIN32_WINNT_MIN < 0x0502 /* < win2003 */ \ + || Z7_WIN32_WINNT_MIN == 0x0502 && !defined(_M_AMD64) +#define Z7_USE_DYN_RegDeleteKeyExW +#endif + +#ifdef Z7_USE_DYN_RegDeleteKeyExW +Z7_DIAGNOSTIC_IGNORE_CAST_FUNCTION typedef LONG (APIENTRY *Func_RegDeleteKeyExW)(HKEY hKey, LPCWSTR lpSubKey, REGSAM samDesired, DWORD Reserved); static Func_RegDeleteKeyExW func_RegDeleteKeyExW; +#endif static WCHAR cmd[MAX_PATH + 4]; static WCHAR cmdError[MAX_PATH + 4]; @@ -247,13 +251,18 @@ static LONG MyRegistry_OpenKey_ReadWrite(HKEY parentKey, LPCWSTR name, HKEY *des static LONG MyRegistry_DeleteKey(HKEY parentKey, LPCWSTR name) { - #if k_Reg_WOW_Flag != 0 - if (func_RegDeleteKeyExW) - return func_RegDeleteKeyExW(parentKey, name, k_Reg_WOW_Flag, 0); - return E_FAIL; - #else +#if k_Reg_WOW_Flag != 0 +#ifdef Z7_USE_DYN_RegDeleteKeyExW + if (!func_RegDeleteKeyExW) + return E_FAIL; + return func_RegDeleteKeyExW +#else + return RegDeleteKeyExW +#endif + (parentKey, name, k_Reg_WOW_Flag, 0); +#else return RegDeleteKeyW(parentKey, name); - #endif +#endif } #ifdef USE_7ZIP_32_DLL @@ -278,13 +287,18 @@ static LONG MyRegistry_OpenKey_ReadWrite_32(HKEY parentKey, LPCWSTR name, HKEY * static LONG MyRegistry_DeleteKey_32(HKEY parentKey, LPCWSTR name) { - #if k_Reg_WOW_Flag_32 != 0 - if (func_RegDeleteKeyExW) - return func_RegDeleteKeyExW(parentKey, name, k_Reg_WOW_Flag_32, 0); - return E_FAIL; - #else +#if k_Reg_WOW_Flag_32 != 0 +#ifdef Z7_USE_DYN_RegDeleteKeyExW + if (!func_RegDeleteKeyExW) + return E_FAIL; + return func_RegDeleteKeyExW +#else + return RegDeleteKeyExW +#endif + (parentKey, name, k_Reg_WOW_Flag_32, 0); +#else return RegDeleteKeyW(parentKey, name); - #endif +#endif } #endif @@ -930,14 +944,17 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, UNUSED_VAR(lpCmdLine) UNUSED_VAR(nCmdShow) - #ifndef UNDER_CE +#ifndef UNDER_CE CoInitialize(NULL); - #endif +#endif - #ifndef UNDER_CE - func_RegDeleteKeyExW = (Func_RegDeleteKeyExW) MY_CAST_FUNC - GetProcAddress(GetModuleHandleW(L"advapi32.dll"), "RegDeleteKeyExW"); - #endif +#ifndef UNDER_CE +#ifdef Z7_USE_DYN_RegDeleteKeyExW + func_RegDeleteKeyExW = + (Func_RegDeleteKeyExW) Z7_CAST_FUNC_C GetProcAddress(GetModuleHandleW(L"advapi32.dll"), + "RegDeleteKeyExW"); +#endif +#endif { const wchar_t *s = GetCommandLineW(); diff --git a/C/Util/7zipUninstall/Precomp.h b/C/Util/7zipUninstall/Precomp.h index bc8fa21..13a41ef 100644 --- a/C/Util/7zipUninstall/Precomp.h +++ b/C/Util/7zipUninstall/Precomp.h @@ -1,14 +1,13 @@ -/* Precomp.h -- StdAfx -2023-03-04 : Igor Pavlov : Public domain */ +/* Precomp.h -- Precomp +2024-01-23 : Igor Pavlov : Public domain */ -#ifndef ZIP7_INC_PRECOMP_H -#define ZIP7_INC_PRECOMP_H +// #ifndef ZIP7_INC_PRECOMP_LOC_H +// #define ZIP7_INC_PRECOMP_LOC_H #if defined(_MSC_VER) && _MSC_VER >= 1800 #pragma warning(disable : 4464) // relative include path contains '..' #endif -#include "../../Compiler.h" -#include "../../7zTypes.h" +#include "../../Precomp.h" -#endif +// #endif diff --git a/C/Util/7zipUninstall/resource.rc b/C/Util/7zipUninstall/resource.rc index 00bdcc0..79400c6 100644 --- a/C/Util/7zipUninstall/resource.rc +++ b/C/Util/7zipUninstall/resource.rc @@ -1,5 +1,6 @@ -#include -#include +#include +// #include +// #include #include #define USE_COPYRIGHT_CR diff --git a/C/Util/Lzma/Precomp.h b/C/Util/Lzma/Precomp.h index bc8fa21..13a41ef 100644 --- a/C/Util/Lzma/Precomp.h +++ b/C/Util/Lzma/Precomp.h @@ -1,14 +1,13 @@ -/* Precomp.h -- StdAfx -2023-03-04 : Igor Pavlov : Public domain */ +/* Precomp.h -- Precomp +2024-01-23 : Igor Pavlov : Public domain */ -#ifndef ZIP7_INC_PRECOMP_H -#define ZIP7_INC_PRECOMP_H +// #ifndef ZIP7_INC_PRECOMP_LOC_H +// #define ZIP7_INC_PRECOMP_LOC_H #if defined(_MSC_VER) && _MSC_VER >= 1800 #pragma warning(disable : 4464) // relative include path contains '..' #endif -#include "../../Compiler.h" -#include "../../7zTypes.h" +#include "../../Precomp.h" -#endif +// #endif diff --git a/C/Util/LzmaLib/Precomp.h b/C/Util/LzmaLib/Precomp.h index bc8fa21..13a41ef 100644 --- a/C/Util/LzmaLib/Precomp.h +++ b/C/Util/LzmaLib/Precomp.h @@ -1,14 +1,13 @@ -/* Precomp.h -- StdAfx -2023-03-04 : Igor Pavlov : Public domain */ +/* Precomp.h -- Precomp +2024-01-23 : Igor Pavlov : Public domain */ -#ifndef ZIP7_INC_PRECOMP_H -#define ZIP7_INC_PRECOMP_H +// #ifndef ZIP7_INC_PRECOMP_LOC_H +// #define ZIP7_INC_PRECOMP_LOC_H #if defined(_MSC_VER) && _MSC_VER >= 1800 #pragma warning(disable : 4464) // relative include path contains '..' #endif -#include "../../Compiler.h" -#include "../../7zTypes.h" +#include "../../Precomp.h" -#endif +// #endif diff --git a/C/Util/LzmaLib/makefile b/C/Util/LzmaLib/makefile index b8e054e..9ed0aa4 100644 --- a/C/Util/LzmaLib/makefile +++ b/C/Util/LzmaLib/makefile @@ -14,16 +14,19 @@ C_OBJS = \ $O\CpuArch.obj \ $O\LzFind.obj \ $O\LzFindMt.obj \ - $O\LzFindOpt.obj \ $O\LzmaDec.obj \ $O\LzmaEnc.obj \ $O\LzmaLib.obj \ $O\Threads.obj \ +!include "../../../CPP/7zip/LzFindOpt.mak" +!include "../../../CPP/7zip/LzmaDec.mak" + OBJS = \ $O\Precomp.obj \ $(LIB_OBJS) \ $(C_OBJS) \ + $(ASM_OBJS) \ $O\resource.res !include "../../../CPP/Build.mak" @@ -52,3 +55,5 @@ $(C_OBJS): ../../$(*B).c $(CCOMPLB_USE) !ENDIF + +!include "../../Asm_c.mak" diff --git a/C/Util/SfxSetup/Precomp.h b/C/Util/SfxSetup/Precomp.h index bc8fa21..13a41ef 100644 --- a/C/Util/SfxSetup/Precomp.h +++ b/C/Util/SfxSetup/Precomp.h @@ -1,14 +1,13 @@ -/* Precomp.h -- StdAfx -2023-03-04 : Igor Pavlov : Public domain */ +/* Precomp.h -- Precomp +2024-01-23 : Igor Pavlov : Public domain */ -#ifndef ZIP7_INC_PRECOMP_H -#define ZIP7_INC_PRECOMP_H +// #ifndef ZIP7_INC_PRECOMP_LOC_H +// #define ZIP7_INC_PRECOMP_LOC_H #if defined(_MSC_VER) && _MSC_VER >= 1800 #pragma warning(disable : 4464) // relative include path contains '..' #endif -#include "../../Compiler.h" -#include "../../7zTypes.h" +#include "../../Precomp.h" -#endif +// #endif diff --git a/C/Util/SfxSetup/SfxSetup.c b/C/Util/SfxSetup/SfxSetup.c index 7304a0b..9b5c1f9 100644 --- a/C/Util/SfxSetup/SfxSetup.c +++ b/C/Util/SfxSetup/SfxSetup.c @@ -1,5 +1,5 @@ /* SfxSetup.c - 7z SFX Setup -2019-02-02 : Igor Pavlov : Public domain */ +2024-01-24 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -278,10 +278,10 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, #ifdef _CONSOLE SetConsoleCtrlHandler(HandlerRoutine, TRUE); #else - UNUSED_VAR(hInstance); - UNUSED_VAR(hPrevInstance); - UNUSED_VAR(lpCmdLine); - UNUSED_VAR(nCmdShow); + UNUSED_VAR(hInstance) + UNUSED_VAR(hPrevInstance) + UNUSED_VAR(lpCmdLine) + UNUSED_VAR(nCmdShow) #endif CrcGenerateTable(); @@ -516,12 +516,13 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, #endif { - const SRes res2 = File_Close(&outFile); + const WRes res2 = File_Close(&outFile); if (res != SZ_OK) break; - if (res2 != SZ_OK) + if (res2 != 0) { - res = res2; + errorMessage = "Can't close output file"; + res = SZ_ERROR_FAIL; break; } } diff --git a/C/Util/SfxSetup/makefile b/C/Util/SfxSetup/makefile index bc0cf8b..b3f25a2 100644 --- a/C/Util/SfxSetup/makefile +++ b/C/Util/SfxSetup/makefile @@ -9,8 +9,6 @@ C_OBJS = \ $O\7zArcIn.obj \ $O\7zBuf.obj \ $O\7zBuf2.obj \ - $O\7zCrc.obj \ - $O\7zCrcOpt.obj \ $O\7zFile.obj \ $O\7zDec.obj \ $O\7zStream.obj \ @@ -27,9 +25,13 @@ C_OBJS = \ 7Z_OBJS = \ $O\SfxSetup.obj \ +!include "../../../CPP/7zip/Crc.mak" +# !include "../../../CPP/7zip/LzmaDec.mak" + OBJS = \ $(7Z_OBJS) \ $(C_OBJS) \ + $(ASM_OBJS) \ $O\resource.res !include "../../../CPP/Build.mak" @@ -38,3 +40,5 @@ $(7Z_OBJS): $(*B).c $(COMPL_O1) $(C_OBJS): ../../$(*B).c $(COMPL_O1) + +!include "../../Asm_c.mak" diff --git a/C/Xxh64.c b/C/Xxh64.c new file mode 100644 index 0000000..dc02a02 --- /dev/null +++ b/C/Xxh64.c @@ -0,0 +1,327 @@ +/* Xxh64.c -- XXH64 hash calculation +original code: Copyright (c) Yann Collet. +2023-08-18 : modified by Igor Pavlov. +This source code is licensed under BSD 2-Clause License. +*/ + +#include "Precomp.h" + +#include "CpuArch.h" +#include "RotateDefs.h" +#include "Xxh64.h" + +#define Z7_XXH_PRIME64_1 UINT64_CONST(0x9E3779B185EBCA87) +#define Z7_XXH_PRIME64_2 UINT64_CONST(0xC2B2AE3D27D4EB4F) +#define Z7_XXH_PRIME64_3 UINT64_CONST(0x165667B19E3779F9) +#define Z7_XXH_PRIME64_4 UINT64_CONST(0x85EBCA77C2B2AE63) +#define Z7_XXH_PRIME64_5 UINT64_CONST(0x27D4EB2F165667C5) + +void Xxh64State_Init(CXxh64State *p) +{ + const UInt64 seed = 0; + p->v[0] = seed + Z7_XXH_PRIME64_1 + Z7_XXH_PRIME64_2; + p->v[1] = seed + Z7_XXH_PRIME64_2; + p->v[2] = seed; + p->v[3] = seed - Z7_XXH_PRIME64_1; +} + +#if !defined(MY_CPU_64BIT) && defined(MY_CPU_X86) && defined(_MSC_VER) + #define Z7_XXH64_USE_ASM +#endif + +#if !defined(MY_CPU_64BIT) && defined(MY_CPU_X86) \ + && defined(Z7_MSC_VER_ORIGINAL) && Z7_MSC_VER_ORIGINAL > 1200 +/* we try to avoid __allmul calls in MSVC for 64-bit multiply. + But MSVC6 still uses __allmul for our code. + So for MSVC6 we use default 64-bit multiply without our optimization. +*/ +#define LOW32(b) ((UInt32)(b & 0xffffffff)) +/* MSVC compiler (MSVC > 1200) can use "mul" instruction + without __allmul for our MY_emulu MACRO. + MY_emulu is similar to __emulu(a, b) MACRO */ +#define MY_emulu(a, b) ((UInt64)(a) * (b)) +#define MY_SET_HIGH32(a) ((UInt64)(a) << 32) +#define MY_MUL32_SET_HIGH32(a, b) MY_SET_HIGH32((UInt32)(a) * (UInt32)(b)) +// /* +#define MY_MUL64(a, b) \ + ( MY_emulu((UInt32)(a), LOW32(b)) + \ + MY_SET_HIGH32( \ + (UInt32)((a) >> 32) * LOW32(b) + \ + (UInt32)(a) * (UInt32)((b) >> 32) \ + )) +// */ +/* +#define MY_MUL64(a, b) \ + ( MY_emulu((UInt32)(a), LOW32(b)) \ + + MY_MUL32_SET_HIGH32((a) >> 32, LOW32(b)) + \ + + MY_MUL32_SET_HIGH32(a, (b) >> 32) \ + ) +*/ + +#define MY_MUL_32_64(a32, b) \ + ( MY_emulu((UInt32)(a32), LOW32(b)) \ + + MY_MUL32_SET_HIGH32(a32, (b) >> 32) \ + ) + +#else +#define MY_MUL64(a, b) ((a) * (b)) +#define MY_MUL_32_64(a32, b) ((a32) * (UInt64)(b)) +#endif + + +static +Z7_FORCE_INLINE +UInt64 Xxh64_Round(UInt64 acc, UInt64 input) +{ + acc += MY_MUL64(input, Z7_XXH_PRIME64_2); + acc = Z7_ROTL64(acc, 31); + return MY_MUL64(acc, Z7_XXH_PRIME64_1); +} + +static UInt64 Xxh64_Merge(UInt64 acc, UInt64 val) +{ + acc ^= Xxh64_Round(0, val); + return MY_MUL64(acc, Z7_XXH_PRIME64_1) + Z7_XXH_PRIME64_4; +} + + +#ifdef Z7_XXH64_USE_ASM + +#define Z7_XXH_PRIME64_1_HIGH 0x9E3779B1 +#define Z7_XXH_PRIME64_1_LOW 0x85EBCA87 +#define Z7_XXH_PRIME64_2_HIGH 0xC2B2AE3D +#define Z7_XXH_PRIME64_2_LOW 0x27D4EB4F + +void +Z7_NO_INLINE +__declspec(naked) +Z7_FASTCALL +Xxh64State_UpdateBlocks(CXxh64State *p, const void *data, const void *end) +{ + #if !defined(__clang__) + UNUSED_VAR(p) + UNUSED_VAR(data) + UNUSED_VAR(end) + #endif + __asm push ebx + __asm push ebp + __asm push esi + __asm push edi + + #define STACK_OFFSET 4 * 8 + __asm sub esp, STACK_OFFSET + +#define COPY_1(n) \ + __asm mov eax, [ecx + n * 4] \ + __asm mov [esp + n * 4], eax \ + +#define COPY_2(n) \ + __asm mov eax, [esp + n * 4] \ + __asm mov [ecx + n * 4], eax \ + + COPY_1(0) + __asm mov edi, [ecx + 1 * 4] \ + COPY_1(2) + COPY_1(3) + COPY_1(4) + COPY_1(5) + COPY_1(6) + COPY_1(7) + + __asm mov esi, edx \ + __asm mov [esp + 0 * 8 + 4], ecx + __asm mov ecx, Z7_XXH_PRIME64_2_LOW \ + __asm mov ebp, Z7_XXH_PRIME64_1_LOW \ + +#define R(n, state1, state1_reg) \ + __asm mov eax, [esi + n * 8] \ + __asm imul ebx, eax, Z7_XXH_PRIME64_2_HIGH \ + __asm add ebx, state1 \ + __asm mul ecx \ + __asm add edx, ebx \ + __asm mov ebx, [esi + n * 8 + 4] \ + __asm imul ebx, ecx \ + __asm add eax, [esp + n * 8] \ + __asm adc edx, ebx \ + __asm mov ebx, eax \ + __asm shld eax, edx, 31 \ + __asm shld edx, ebx, 31 \ + __asm imul state1_reg, eax, Z7_XXH_PRIME64_1_HIGH \ + __asm imul edx, ebp \ + __asm add state1_reg, edx \ + __asm mul ebp \ + __asm add state1_reg, edx \ + __asm mov [esp + n * 8], eax \ + +#define R2(n) \ + R(n, [esp + n * 8 + 4], ebx) \ + __asm mov [esp + n * 8 + 4], ebx \ + + __asm align 16 + __asm main_loop: + R(0, edi, edi) + R2(1) + R2(2) + R2(3) + __asm add esi, 32 + __asm cmp esi, [esp + STACK_OFFSET + 4 * 4 + 4] + __asm jne main_loop + + __asm mov ecx, [esp + 0 * 8 + 4] + + COPY_2(0) + __asm mov [ecx + 1 * 4], edi + COPY_2(2) + COPY_2(3) + COPY_2(4) + COPY_2(5) + COPY_2(6) + COPY_2(7) + + __asm add esp, STACK_OFFSET + __asm pop edi + __asm pop esi + __asm pop ebp + __asm pop ebx + __asm ret 4 +} + +#else + +void +Z7_NO_INLINE +Z7_FASTCALL +Xxh64State_UpdateBlocks(CXxh64State *p, const void *_data, const void *end) +{ + const Byte *data = (const Byte *)_data; + UInt64 v[4]; + v[0] = p->v[0]; + v[1] = p->v[1]; + v[2] = p->v[2]; + v[3] = p->v[3]; + do + { + v[0] = Xxh64_Round(v[0], GetUi64(data)); data += 8; + v[1] = Xxh64_Round(v[1], GetUi64(data)); data += 8; + v[2] = Xxh64_Round(v[2], GetUi64(data)); data += 8; + v[3] = Xxh64_Round(v[3], GetUi64(data)); data += 8; + } + while (data != end); + p->v[0] = v[0]; + p->v[1] = v[1]; + p->v[2] = v[2]; + p->v[3] = v[3]; +} + +#endif + +UInt64 Xxh64State_Digest(const CXxh64State *p, const void *_data, UInt64 count) +{ + UInt64 h = p->v[2]; + + if (count >= 32) + { + h = Z7_ROTL64(p->v[0], 1) + + Z7_ROTL64(p->v[1], 7) + + Z7_ROTL64(h, 12) + + Z7_ROTL64(p->v[3], 18); + h = Xxh64_Merge(h, p->v[0]); + h = Xxh64_Merge(h, p->v[1]); + h = Xxh64_Merge(h, p->v[2]); + h = Xxh64_Merge(h, p->v[3]); + } + else + h += Z7_XXH_PRIME64_5; + + h += count; + + // XXH64_finalize(): + { + unsigned cnt = (unsigned)count & 31; + const Byte *data = (const Byte *)_data; + while (cnt >= 8) + { + h ^= Xxh64_Round(0, GetUi64(data)); + data += 8; + h = Z7_ROTL64(h, 27); + h = MY_MUL64(h, Z7_XXH_PRIME64_1) + Z7_XXH_PRIME64_4; + cnt -= 8; + } + if (cnt >= 4) + { + const UInt32 v = GetUi32(data); + data += 4; + h ^= MY_MUL_32_64(v, Z7_XXH_PRIME64_1); + h = Z7_ROTL64(h, 23); + h = MY_MUL64(h, Z7_XXH_PRIME64_2) + Z7_XXH_PRIME64_3; + cnt -= 4; + } + while (cnt) + { + const UInt32 v = *data++; + h ^= MY_MUL_32_64(v, Z7_XXH_PRIME64_5); + h = Z7_ROTL64(h, 11); + h = MY_MUL64(h, Z7_XXH_PRIME64_1); + cnt--; + } + // XXH64_avalanche(h): + h ^= h >> 33; h = MY_MUL64(h, Z7_XXH_PRIME64_2); + h ^= h >> 29; h = MY_MUL64(h, Z7_XXH_PRIME64_3); + h ^= h >> 32; + return h; + } +} + + +void Xxh64_Init(CXxh64 *p) +{ + Xxh64State_Init(&p->state); + p->count = 0; + p->buf64[0] = 0; + p->buf64[1] = 0; + p->buf64[2] = 0; + p->buf64[3] = 0; +} + +void Xxh64_Update(CXxh64 *p, const void *_data, size_t size) +{ + const Byte *data = (const Byte *)_data; + unsigned cnt; + if (size == 0) + return; + cnt = (unsigned)p->count; + p->count += size; + + if (cnt &= 31) + { + unsigned rem = 32 - cnt; + Byte *dest = (Byte *)p->buf64 + cnt; + if (rem > size) + rem = (unsigned)size; + size -= rem; + cnt += rem; + // memcpy((Byte *)p->buf64 + cnt, data, rem); + do + *dest++ = *data++; + while (--rem); + if (cnt != 32) + return; + Xxh64State_UpdateBlocks(&p->state, p->buf64, &p->buf64[4]); + } + + if (size &= ~(size_t)31) + { + Xxh64State_UpdateBlocks(&p->state, data, data + size); + data += size; + } + + cnt = (unsigned)p->count & 31; + if (cnt) + { + // memcpy(p->buf64, data, cnt); + Byte *dest = (Byte *)p->buf64; + do + *dest++ = *data++; + while (--cnt); + } +} diff --git a/C/Xxh64.h b/C/Xxh64.h new file mode 100644 index 0000000..efef65e --- /dev/null +++ b/C/Xxh64.h @@ -0,0 +1,50 @@ +/* Xxh64.h -- XXH64 hash calculation interfaces +2023-08-18 : Igor Pavlov : Public domain */ + +#ifndef ZIP7_INC_XXH64_H +#define ZIP7_INC_XXH64_H + +#include "7zTypes.h" + +EXTERN_C_BEGIN + +#define Z7_XXH64_BLOCK_SIZE (4 * 8) + +typedef struct +{ + UInt64 v[4]; +} CXxh64State; + +void Xxh64State_Init(CXxh64State *p); + +// end != data && end == data + Z7_XXH64_BLOCK_SIZE * numBlocks +void Z7_FASTCALL Xxh64State_UpdateBlocks(CXxh64State *p, const void *data, const void *end); + +/* +Xxh64State_Digest(): +data: + the function processes only + (totalCount & (Z7_XXH64_BLOCK_SIZE - 1)) bytes in (data): (smaller than 32 bytes). +totalCount: total size of hashed stream: + it includes total size of data processed by previous Xxh64State_UpdateBlocks() calls, + and it also includes current processed size in (data). +*/ +UInt64 Xxh64State_Digest(const CXxh64State *p, const void *data, UInt64 totalCount); + + +typedef struct +{ + CXxh64State state; + UInt64 count; + UInt64 buf64[4]; +} CXxh64; + +void Xxh64_Init(CXxh64 *p); +void Xxh64_Update(CXxh64 *p, const void *data, size_t size); + +#define Xxh64_Digest(p) \ + Xxh64State_Digest(&(p)->state, (p)->buf64, (p)->count) + +EXTERN_C_END + +#endif diff --git a/C/Xz.c b/C/Xz.c index 4ad0710..d07550d 100644 --- a/C/Xz.c +++ b/C/Xz.c @@ -1,5 +1,5 @@ /* Xz.c - Xz -2023-04-02 : Igor Pavlov : Public domain */ +2024-03-01 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -52,6 +52,7 @@ void XzCheck_Init(CXzCheck *p, unsigned mode) case XZ_CHECK_CRC32: p->crc = CRC_INIT_VAL; break; case XZ_CHECK_CRC64: p->crc64 = CRC64_INIT_VAL; break; case XZ_CHECK_SHA256: Sha256_Init(&p->sha); break; + default: break; } } @@ -62,6 +63,7 @@ void XzCheck_Update(CXzCheck *p, const void *data, size_t size) case XZ_CHECK_CRC32: p->crc = CrcUpdate(p->crc, data, size); break; case XZ_CHECK_CRC64: p->crc64 = Crc64Update(p->crc64, data, size); break; case XZ_CHECK_SHA256: Sha256_Update(&p->sha, (const Byte *)data, size); break; + default: break; } } diff --git a/C/Xz.h b/C/Xz.h index d5001f6..42bc685 100644 --- a/C/Xz.h +++ b/C/Xz.h @@ -1,5 +1,5 @@ /* Xz.h - Xz interface -2023-04-13 : Igor Pavlov : Public domain */ +2024-01-26 : Igor Pavlov : Public domain */ #ifndef ZIP7_INC_XZ_H #define ZIP7_INC_XZ_H @@ -18,6 +18,7 @@ EXTERN_C_BEGIN #define XZ_ID_ARMT 8 #define XZ_ID_SPARC 9 #define XZ_ID_ARM64 0xa +#define XZ_ID_RISCV 0xb #define XZ_ID_LZMA2 0x21 unsigned Xz_ReadVarInt(const Byte *p, size_t maxSize, UInt64 *value); @@ -233,13 +234,13 @@ typedef enum typedef struct { EXzState state; - UInt32 pos; + unsigned pos; unsigned alignPos; unsigned indexPreSize; CXzStreamFlags streamFlags; - UInt32 blockHeaderSize; + unsigned blockHeaderSize; UInt64 packSize; UInt64 unpackSize; diff --git a/C/XzCrc64.c b/C/XzCrc64.c index c2fad6c..94fc1af 100644 --- a/C/XzCrc64.c +++ b/C/XzCrc64.c @@ -1,5 +1,5 @@ /* XzCrc64.c -- CRC64 calculation -2023-04-02 : Igor Pavlov : Public domain */ +2023-12-08 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -8,36 +8,76 @@ #define kCrc64Poly UINT64_CONST(0xC96C5795D7870F42) -#ifdef MY_CPU_LE - #define CRC64_NUM_TABLES 4 +// for debug only : define Z7_CRC64_DEBUG_BE to test big-endian code in little-endian cpu +// #define Z7_CRC64_DEBUG_BE +#ifdef Z7_CRC64_DEBUG_BE +#undef MY_CPU_LE +#define MY_CPU_BE +#endif + +#ifdef Z7_CRC64_NUM_TABLES + #define Z7_CRC64_NUM_TABLES_USE Z7_CRC64_NUM_TABLES #else - #define CRC64_NUM_TABLES 5 + #define Z7_CRC64_NUM_TABLES_USE 12 +#endif - UInt64 Z7_FASTCALL XzCrc64UpdateT1_BeT4(UInt64 v, const void *data, size_t size, const UInt64 *table); +#if Z7_CRC64_NUM_TABLES_USE < 1 + #error Stop_Compiling_Bad_Z7_CRC_NUM_TABLES #endif + +#if Z7_CRC64_NUM_TABLES_USE != 1 + #ifndef MY_CPU_BE - UInt64 Z7_FASTCALL XzCrc64UpdateT4(UInt64 v, const void *data, size_t size, const UInt64 *table); + #define FUNC_NAME_LE_2(s) XzCrc64UpdateT ## s + #define FUNC_NAME_LE_1(s) FUNC_NAME_LE_2(s) + #define FUNC_NAME_LE FUNC_NAME_LE_1(Z7_CRC64_NUM_TABLES_USE) + UInt64 Z7_FASTCALL FUNC_NAME_LE (UInt64 v, const void *data, size_t size, const UInt64 *table); +#endif +#ifndef MY_CPU_LE + #define FUNC_NAME_BE_2(s) XzCrc64UpdateBeT ## s + #define FUNC_NAME_BE_1(s) FUNC_NAME_BE_2(s) + #define FUNC_NAME_BE FUNC_NAME_BE_1(Z7_CRC64_NUM_TABLES_USE) + UInt64 Z7_FASTCALL FUNC_NAME_BE (UInt64 v, const void *data, size_t size, const UInt64 *table); #endif -typedef UInt64 (Z7_FASTCALL *CRC64_FUNC)(UInt64 v, const void *data, size_t size, const UInt64 *table); +#if defined(MY_CPU_LE) + #define FUNC_REF FUNC_NAME_LE +#elif defined(MY_CPU_BE) + #define FUNC_REF FUNC_NAME_BE +#else + #define FUNC_REF g_Crc64Update + static UInt64 (Z7_FASTCALL *FUNC_REF)(UInt64 v, const void *data, size_t size, const UInt64 *table); +#endif + +#endif + + +MY_ALIGN(64) +static UInt64 g_Crc64Table[256 * Z7_CRC64_NUM_TABLES_USE]; -static CRC64_FUNC g_Crc64Update; -UInt64 g_Crc64Table[256 * CRC64_NUM_TABLES]; UInt64 Z7_FASTCALL Crc64Update(UInt64 v, const void *data, size_t size) { - return g_Crc64Update(v, data, size, g_Crc64Table); +#if Z7_CRC64_NUM_TABLES_USE == 1 + #define CRC64_UPDATE_BYTE_2(crc, b) (table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8)) + const UInt64 *table = g_Crc64Table; + const Byte *p = (const Byte *)data; + const Byte *lim = p + size; + for (; p != lim; p++) + v = CRC64_UPDATE_BYTE_2(v, *p); + return v; + #undef CRC64_UPDATE_BYTE_2 +#else + return FUNC_REF (v, data, size, g_Crc64Table); +#endif } -UInt64 Z7_FASTCALL Crc64Calc(const void *data, size_t size) -{ - return g_Crc64Update(CRC64_INIT_VAL, data, size, g_Crc64Table) ^ CRC64_INIT_VAL; -} +Z7_NO_INLINE void Z7_FASTCALL Crc64GenerateTable(void) { - UInt32 i; + unsigned i; for (i = 0; i < 256; i++) { UInt64 r = i; @@ -46,35 +86,55 @@ void Z7_FASTCALL Crc64GenerateTable(void) r = (r >> 1) ^ (kCrc64Poly & ((UInt64)0 - (r & 1))); g_Crc64Table[i] = r; } - for (i = 256; i < 256 * CRC64_NUM_TABLES; i++) + +#if Z7_CRC64_NUM_TABLES_USE != 1 +#if 1 || 1 && defined(MY_CPU_X86) // low register count + for (i = 0; i < 256 * (Z7_CRC64_NUM_TABLES_USE - 1); i++) { - const UInt64 r = g_Crc64Table[(size_t)i - 256]; - g_Crc64Table[i] = g_Crc64Table[r & 0xFF] ^ (r >> 8); + const UInt64 r0 = g_Crc64Table[(size_t)i]; + g_Crc64Table[(size_t)i + 256] = g_Crc64Table[(Byte)r0] ^ (r0 >> 8); } - - #ifdef MY_CPU_LE - - g_Crc64Update = XzCrc64UpdateT4; +#else + for (i = 0; i < 256 * (Z7_CRC64_NUM_TABLES_USE - 1); i += 2) + { + UInt64 r0 = g_Crc64Table[(size_t)(i) ]; + UInt64 r1 = g_Crc64Table[(size_t)(i) + 1]; + r0 = g_Crc64Table[(Byte)r0] ^ (r0 >> 8); + r1 = g_Crc64Table[(Byte)r1] ^ (r1 >> 8); + g_Crc64Table[(size_t)i + 256 ] = r0; + g_Crc64Table[(size_t)i + 256 + 1] = r1; + } +#endif - #else +#ifndef MY_CPU_LE { - #ifndef MY_CPU_BE +#ifndef MY_CPU_BE UInt32 k = 1; if (*(const Byte *)&k == 1) - g_Crc64Update = XzCrc64UpdateT4; + FUNC_REF = FUNC_NAME_LE; else - #endif +#endif { - for (i = 256 * CRC64_NUM_TABLES - 1; i >= 256; i--) +#ifndef MY_CPU_BE + FUNC_REF = FUNC_NAME_BE; +#endif + for (i = 0; i < 256 * Z7_CRC64_NUM_TABLES_USE; i++) { - const UInt64 x = g_Crc64Table[(size_t)i - 256]; + const UInt64 x = g_Crc64Table[i]; g_Crc64Table[i] = Z7_BSWAP64(x); } - g_Crc64Update = XzCrc64UpdateT1_BeT4; } } - #endif +#endif // ndef MY_CPU_LE +#endif // Z7_CRC64_NUM_TABLES_USE != 1 } #undef kCrc64Poly -#undef CRC64_NUM_TABLES +#undef Z7_CRC64_NUM_TABLES_USE +#undef FUNC_REF +#undef FUNC_NAME_LE_2 +#undef FUNC_NAME_LE_1 +#undef FUNC_NAME_LE +#undef FUNC_NAME_BE_2 +#undef FUNC_NAME_BE_1 +#undef FUNC_NAME_BE diff --git a/C/XzCrc64.h b/C/XzCrc64.h index ca46869..04f8153 100644 --- a/C/XzCrc64.h +++ b/C/XzCrc64.h @@ -1,5 +1,5 @@ /* XzCrc64.h -- CRC64 calculation -2023-04-02 : Igor Pavlov : Public domain */ +2023-12-08 : Igor Pavlov : Public domain */ #ifndef ZIP7_INC_XZ_CRC64_H #define ZIP7_INC_XZ_CRC64_H @@ -10,16 +10,16 @@ EXTERN_C_BEGIN -extern UInt64 g_Crc64Table[]; +// extern UInt64 g_Crc64Table[]; void Z7_FASTCALL Crc64GenerateTable(void); #define CRC64_INIT_VAL UINT64_CONST(0xFFFFFFFFFFFFFFFF) #define CRC64_GET_DIGEST(crc) ((crc) ^ CRC64_INIT_VAL) -#define CRC64_UPDATE_BYTE(crc, b) (g_Crc64Table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8)) +// #define CRC64_UPDATE_BYTE(crc, b) (g_Crc64Table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8)) UInt64 Z7_FASTCALL Crc64Update(UInt64 crc, const void *data, size_t size); -UInt64 Z7_FASTCALL Crc64Calc(const void *data, size_t size); +// UInt64 Z7_FASTCALL Crc64Calc(const void *data, size_t size); EXTERN_C_END diff --git a/C/XzCrc64Opt.c b/C/XzCrc64Opt.c index d03374c..0c1fc2f 100644 --- a/C/XzCrc64Opt.c +++ b/C/XzCrc64Opt.c @@ -1,61 +1,261 @@ -/* XzCrc64Opt.c -- CRC64 calculation -2023-04-02 : Igor Pavlov : Public domain */ +/* XzCrc64Opt.c -- CRC64 calculation (optimized functions) +2023-12-08 : Igor Pavlov : Public domain */ #include "Precomp.h" #include "CpuArch.h" +#if !defined(Z7_CRC64_NUM_TABLES) || Z7_CRC64_NUM_TABLES > 1 + +// for debug only : define Z7_CRC64_DEBUG_BE to test big-endian code in little-endian cpu +// #define Z7_CRC64_DEBUG_BE +#ifdef Z7_CRC64_DEBUG_BE +#undef MY_CPU_LE +#define MY_CPU_BE +#endif + +#if defined(MY_CPU_64BIT) +#define Z7_CRC64_USE_64BIT +#endif + +// the value Z7_CRC64_NUM_TABLES_USE must be defined to same value as in XzCrc64.c +#ifdef Z7_CRC64_NUM_TABLES +#define Z7_CRC64_NUM_TABLES_USE Z7_CRC64_NUM_TABLES +#else +#define Z7_CRC64_NUM_TABLES_USE 12 +#endif + +#if Z7_CRC64_NUM_TABLES_USE % 4 || \ + Z7_CRC64_NUM_TABLES_USE < 4 || \ + Z7_CRC64_NUM_TABLES_USE > 4 * 4 + #error Stop_Compiling_Bad_CRC64_NUM_TABLES +#endif + + #ifndef MY_CPU_BE -#define CRC64_UPDATE_BYTE_2(crc, b) (table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8)) +#define CRC64_UPDATE_BYTE_2(crc, b) (table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8)) + +#if defined(Z7_CRC64_USE_64BIT) && (Z7_CRC64_NUM_TABLES_USE % 8 == 0) -UInt64 Z7_FASTCALL XzCrc64UpdateT4(UInt64 v, const void *data, size_t size, const UInt64 *table); -UInt64 Z7_FASTCALL XzCrc64UpdateT4(UInt64 v, const void *data, size_t size, const UInt64 *table) +#define Q64LE(n, d) \ + ( (table + ((n) * 8 + 7) * 0x100)[((d) ) & 0xFF] \ + ^ (table + ((n) * 8 + 6) * 0x100)[((d) >> 1 * 8) & 0xFF] \ + ^ (table + ((n) * 8 + 5) * 0x100)[((d) >> 2 * 8) & 0xFF] \ + ^ (table + ((n) * 8 + 4) * 0x100)[((d) >> 3 * 8) & 0xFF] \ + ^ (table + ((n) * 8 + 3) * 0x100)[((d) >> 4 * 8) & 0xFF] \ + ^ (table + ((n) * 8 + 2) * 0x100)[((d) >> 5 * 8) & 0xFF] \ + ^ (table + ((n) * 8 + 1) * 0x100)[((d) >> 6 * 8) & 0xFF] \ + ^ (table + ((n) * 8 + 0) * 0x100)[((d) >> 7 * 8)] ) + +#define R64(a) *((const UInt64 *)(const void *)p + (a)) + +#else + +#define Q32LE(n, d) \ + ( (table + ((n) * 4 + 3) * 0x100)[((d) ) & 0xFF] \ + ^ (table + ((n) * 4 + 2) * 0x100)[((d) >> 1 * 8) & 0xFF] \ + ^ (table + ((n) * 4 + 1) * 0x100)[((d) >> 2 * 8) & 0xFF] \ + ^ (table + ((n) * 4 + 0) * 0x100)[((d) >> 3 * 8)] ) + +#define R32(a) *((const UInt32 *)(const void *)p + (a)) + +#endif + + +#define CRC64_FUNC_PRE_LE2(step) \ +UInt64 Z7_FASTCALL XzCrc64UpdateT ## step (UInt64 v, const void *data, size_t size, const UInt64 *table) + +#define CRC64_FUNC_PRE_LE(step) \ + CRC64_FUNC_PRE_LE2(step); \ + CRC64_FUNC_PRE_LE2(step) + +CRC64_FUNC_PRE_LE(Z7_CRC64_NUM_TABLES_USE) { const Byte *p = (const Byte *)data; - for (; size > 0 && ((unsigned)(ptrdiff_t)p & 3) != 0; size--, p++) + const Byte *lim; + for (; size && ((unsigned)(ptrdiff_t)p & (7 - (Z7_CRC64_NUM_TABLES_USE & 4))) != 0; size--, p++) v = CRC64_UPDATE_BYTE_2(v, *p); - for (; size >= 4; size -= 4, p += 4) + lim = p + size; + if (size >= Z7_CRC64_NUM_TABLES_USE) { - const UInt32 d = (UInt32)v ^ *(const UInt32 *)(const void *)p; - v = (v >> 32) - ^ (table + 0x300)[((d ) & 0xFF)] - ^ (table + 0x200)[((d >> 8) & 0xFF)] - ^ (table + 0x100)[((d >> 16) & 0xFF)] - ^ (table + 0x000)[((d >> 24))]; + lim -= Z7_CRC64_NUM_TABLES_USE; + do + { +#if Z7_CRC64_NUM_TABLES_USE == 4 + const UInt32 d = (UInt32)v ^ R32(0); + v = (v >> 32) ^ Q32LE(0, d); +#elif Z7_CRC64_NUM_TABLES_USE == 8 +#ifdef Z7_CRC64_USE_64BIT + v ^= R64(0); + v = Q64LE(0, v); +#else + UInt32 v0, v1; + v0 = (UInt32)v ^ R32(0); + v1 = (UInt32)(v >> 32) ^ R32(1); + v = Q32LE(1, v0) ^ Q32LE(0, v1); +#endif +#elif Z7_CRC64_NUM_TABLES_USE == 12 + UInt32 w; + UInt32 v0, v1; + v0 = (UInt32)v ^ R32(0); + v1 = (UInt32)(v >> 32) ^ R32(1); + w = R32(2); + v = Q32LE(0, w); + v ^= Q32LE(2, v0) ^ Q32LE(1, v1); +#elif Z7_CRC64_NUM_TABLES_USE == 16 +#ifdef Z7_CRC64_USE_64BIT + UInt64 w; + UInt64 x; + w = R64(1); x = Q64LE(0, w); + v ^= R64(0); v = x ^ Q64LE(1, v); +#else + UInt32 v0, v1; + UInt32 r0, r1; + v0 = (UInt32)v ^ R32(0); + v1 = (UInt32)(v >> 32) ^ R32(1); + r0 = R32(2); + r1 = R32(3); + v = Q32LE(1, r0) ^ Q32LE(0, r1); + v ^= Q32LE(3, v0) ^ Q32LE(2, v1); +#endif +#else +#error Stop_Compiling_Bad_CRC64_NUM_TABLES +#endif + p += Z7_CRC64_NUM_TABLES_USE; + } + while (p <= lim); + lim += Z7_CRC64_NUM_TABLES_USE; } - for (; size > 0; size--, p++) + for (; p < lim; p++) v = CRC64_UPDATE_BYTE_2(v, *p); return v; } +#undef CRC64_UPDATE_BYTE_2 +#undef R32 +#undef R64 +#undef Q32LE +#undef Q64LE +#undef CRC64_FUNC_PRE_LE +#undef CRC64_FUNC_PRE_LE2 + #endif + + #ifndef MY_CPU_LE -#define CRC64_UPDATE_BYTE_2_BE(crc, b) (table[(Byte)((crc) >> 56) ^ (b)] ^ ((crc) << 8)) +#define CRC64_UPDATE_BYTE_2_BE(crc, b) (table[((crc) >> 56) ^ (b)] ^ ((crc) << 8)) + +#if defined(Z7_CRC64_USE_64BIT) && (Z7_CRC64_NUM_TABLES_USE % 8 == 0) + +#define Q64BE(n, d) \ + ( (table + ((n) * 8 + 0) * 0x100)[(Byte)(d)] \ + ^ (table + ((n) * 8 + 1) * 0x100)[((d) >> 1 * 8) & 0xFF] \ + ^ (table + ((n) * 8 + 2) * 0x100)[((d) >> 2 * 8) & 0xFF] \ + ^ (table + ((n) * 8 + 3) * 0x100)[((d) >> 3 * 8) & 0xFF] \ + ^ (table + ((n) * 8 + 4) * 0x100)[((d) >> 4 * 8) & 0xFF] \ + ^ (table + ((n) * 8 + 5) * 0x100)[((d) >> 5 * 8) & 0xFF] \ + ^ (table + ((n) * 8 + 6) * 0x100)[((d) >> 6 * 8) & 0xFF] \ + ^ (table + ((n) * 8 + 7) * 0x100)[((d) >> 7 * 8)] ) + +#ifdef Z7_CRC64_DEBUG_BE + #define R64BE(a) GetBe64a((const UInt64 *)(const void *)p + (a)) +#else + #define R64BE(a) *((const UInt64 *)(const void *)p + (a)) +#endif + +#else + +#define Q32BE(n, d) \ + ( (table + ((n) * 4 + 0) * 0x100)[(Byte)(d)] \ + ^ (table + ((n) * 4 + 1) * 0x100)[((d) >> 1 * 8) & 0xFF] \ + ^ (table + ((n) * 4 + 2) * 0x100)[((d) >> 2 * 8) & 0xFF] \ + ^ (table + ((n) * 4 + 3) * 0x100)[((d) >> 3 * 8)] ) -UInt64 Z7_FASTCALL XzCrc64UpdateT1_BeT4(UInt64 v, const void *data, size_t size, const UInt64 *table); -UInt64 Z7_FASTCALL XzCrc64UpdateT1_BeT4(UInt64 v, const void *data, size_t size, const UInt64 *table) +#ifdef Z7_CRC64_DEBUG_BE + #define R32BE(a) GetBe32a((const UInt32 *)(const void *)p + (a)) +#else + #define R32BE(a) *((const UInt32 *)(const void *)p + (a)) +#endif + +#endif + +#define CRC64_FUNC_PRE_BE2(step) \ +UInt64 Z7_FASTCALL XzCrc64UpdateBeT ## step (UInt64 v, const void *data, size_t size, const UInt64 *table) + +#define CRC64_FUNC_PRE_BE(step) \ + CRC64_FUNC_PRE_BE2(step); \ + CRC64_FUNC_PRE_BE2(step) + +CRC64_FUNC_PRE_BE(Z7_CRC64_NUM_TABLES_USE) { const Byte *p = (const Byte *)data; - table += 0x100; + const Byte *lim; v = Z7_BSWAP64(v); - for (; size > 0 && ((unsigned)(ptrdiff_t)p & 3) != 0; size--, p++) + for (; size && ((unsigned)(ptrdiff_t)p & (7 - (Z7_CRC64_NUM_TABLES_USE & 4))) != 0; size--, p++) v = CRC64_UPDATE_BYTE_2_BE(v, *p); - for (; size >= 4; size -= 4, p += 4) + lim = p + size; + if (size >= Z7_CRC64_NUM_TABLES_USE) { - const UInt32 d = (UInt32)(v >> 32) ^ *(const UInt32 *)(const void *)p; - v = (v << 32) - ^ (table + 0x000)[((d ) & 0xFF)] - ^ (table + 0x100)[((d >> 8) & 0xFF)] - ^ (table + 0x200)[((d >> 16) & 0xFF)] - ^ (table + 0x300)[((d >> 24))]; + lim -= Z7_CRC64_NUM_TABLES_USE; + do + { +#if Z7_CRC64_NUM_TABLES_USE == 4 + const UInt32 d = (UInt32)(v >> 32) ^ R32BE(0); + v = (v << 32) ^ Q32BE(0, d); +#elif Z7_CRC64_NUM_TABLES_USE == 12 + const UInt32 d1 = (UInt32)(v >> 32) ^ R32BE(0); + const UInt32 d0 = (UInt32)(v ) ^ R32BE(1); + const UInt32 w = R32BE(2); + v = Q32BE(0, w); + v ^= Q32BE(2, d1) ^ Q32BE(1, d0); + +#elif Z7_CRC64_NUM_TABLES_USE == 8 + #ifdef Z7_CRC64_USE_64BIT + v ^= R64BE(0); + v = Q64BE(0, v); + #else + const UInt32 d1 = (UInt32)(v >> 32) ^ R32BE(0); + const UInt32 d0 = (UInt32)(v ) ^ R32BE(1); + v = Q32BE(1, d1) ^ Q32BE(0, d0); + #endif +#elif Z7_CRC64_NUM_TABLES_USE == 16 + #ifdef Z7_CRC64_USE_64BIT + const UInt64 w = R64BE(1); + v ^= R64BE(0); + v = Q64BE(0, w) ^ Q64BE(1, v); + #else + const UInt32 d1 = (UInt32)(v >> 32) ^ R32BE(0); + const UInt32 d0 = (UInt32)(v ) ^ R32BE(1); + const UInt32 w1 = R32BE(2); + const UInt32 w0 = R32BE(3); + v = Q32BE(1, w1) ^ Q32BE(0, w0); + v ^= Q32BE(3, d1) ^ Q32BE(2, d0); + #endif +#elif +#error Stop_Compiling_Bad_CRC64_NUM_TABLES +#endif + p += Z7_CRC64_NUM_TABLES_USE; + } + while (p <= lim); + lim += Z7_CRC64_NUM_TABLES_USE; } - for (; size > 0; size--, p++) + for (; p < lim; p++) v = CRC64_UPDATE_BYTE_2_BE(v, *p); return Z7_BSWAP64(v); } +#undef CRC64_UPDATE_BYTE_2_BE +#undef R32BE +#undef R64BE +#undef Q32BE +#undef Q64BE +#undef CRC64_FUNC_PRE_BE +#undef CRC64_FUNC_PRE_BE2 + +#endif +#undef Z7_CRC64_NUM_TABLES_USE #endif diff --git a/C/XzDec.c b/C/XzDec.c index a5f7039..3d1c98e 100644 --- a/C/XzDec.c +++ b/C/XzDec.c @@ -1,5 +1,5 @@ /* XzDec.c -- Xz Decode -2023-04-13 : Igor Pavlov : Public domain */ +2024-03-01 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -105,30 +105,32 @@ static SRes XzBcFilterState_SetProps(void *pp, const Byte *props, size_t propSiz { if (propSize != 1) return SZ_ERROR_UNSUPPORTED; - p->delta = (unsigned)props[0] + 1; + p->delta = (UInt32)props[0] + 1; } else { if (propSize == 4) { - UInt32 v = GetUi32(props); + const UInt32 v = GetUi32(props); switch (p->methodId) { case XZ_ID_PPC: case XZ_ID_ARM: case XZ_ID_SPARC: case XZ_ID_ARM64: - if ((v & 3) != 0) + if (v & 3) return SZ_ERROR_UNSUPPORTED; break; case XZ_ID_ARMT: - if ((v & 1) != 0) + case XZ_ID_RISCV: + if (v & 1) return SZ_ERROR_UNSUPPORTED; break; case XZ_ID_IA64: - if ((v & 0xF) != 0) + if (v & 0xf) return SZ_ERROR_UNSUPPORTED; break; + default: break; } p->ip = v; } @@ -151,12 +153,13 @@ static void XzBcFilterState_Init(void *pp) static const z7_Func_BranchConv g_Funcs_BranchConv_RISC_Dec[] = { - Z7_BRANCH_CONV_DEC(PPC), - Z7_BRANCH_CONV_DEC(IA64), - Z7_BRANCH_CONV_DEC(ARM), - Z7_BRANCH_CONV_DEC(ARMT), - Z7_BRANCH_CONV_DEC(SPARC), - Z7_BRANCH_CONV_DEC(ARM64) + Z7_BRANCH_CONV_DEC_2 (BranchConv_PPC), + Z7_BRANCH_CONV_DEC_2 (BranchConv_IA64), + Z7_BRANCH_CONV_DEC_2 (BranchConv_ARM), + Z7_BRANCH_CONV_DEC_2 (BranchConv_ARMT), + Z7_BRANCH_CONV_DEC_2 (BranchConv_SPARC), + Z7_BRANCH_CONV_DEC_2 (BranchConv_ARM64), + Z7_BRANCH_CONV_DEC_2 (BranchConv_RISCV) }; static SizeT XzBcFilterStateBase_Filter_Dec(CXzBcFilterStateBase *p, Byte *data, SizeT size) @@ -262,7 +265,7 @@ static SRes XzBcFilterState_Code2(void *pp, #define XZ_IS_SUPPORTED_FILTER_ID(id) \ - ((id) >= XZ_ID_Delta && (id) <= XZ_ID_ARM64) + ((id) >= XZ_ID_Delta && (id) <= XZ_ID_RISCV) SRes Xz_StateCoder_Bc_SetFromMethod_Func(IStateCoder *p, UInt64 id, Xz_Func_BcFilterStateBase_Filter func, ISzAllocPtr alloc) @@ -541,13 +544,12 @@ static SRes MixCoder_SetFromMethod(CMixCoder *p, unsigned coderIndex, UInt64 met { IStateCoder *sc = &p->coders[coderIndex]; p->ids[coderIndex] = methodId; - switch (methodId) - { - case XZ_ID_LZMA2: return Lzma2State_SetFromMethod(sc, outBuf, outBufSize, p->alloc); - #ifdef USE_SUBBLOCK - case XZ_ID_Subblock: return SbState_SetFromMethod(sc, p->alloc); - #endif - } + if (methodId == XZ_ID_LZMA2) + return Lzma2State_SetFromMethod(sc, outBuf, outBufSize, p->alloc); +#ifdef USE_SUBBLOCK + if (methodId == XZ_ID_Subblock) + return SbState_SetFromMethod(sc, p->alloc); +#endif if (coderIndex == 0) return SZ_ERROR_UNSUPPORTED; return Xz_StateCoder_Bc_SetFromMethod_Func(sc, methodId, @@ -558,10 +560,8 @@ static SRes MixCoder_SetFromMethod(CMixCoder *p, unsigned coderIndex, UInt64 met static SRes MixCoder_ResetFromMethod(CMixCoder *p, unsigned coderIndex, UInt64 methodId, Byte *outBuf, size_t outBufSize) { IStateCoder *sc = &p->coders[coderIndex]; - switch (methodId) - { - case XZ_ID_LZMA2: return Lzma2State_ResetOutBuf(sc, outBuf, outBufSize); - } + if (methodId == XZ_ID_LZMA2) + return Lzma2State_ResetOutBuf(sc, outBuf, outBufSize); return SZ_ERROR_UNSUPPORTED; } @@ -804,7 +804,7 @@ static BoolInt Xz_CheckFooter(CXzStreamFlags flags, UInt64 indexSize, const Byte } #define READ_VARINT_AND_CHECK(buf, pos, size, res) \ - { unsigned s = Xz_ReadVarInt(buf + pos, size - pos, res); \ + { const unsigned s = Xz_ReadVarInt(buf + pos, size - pos, res); \ if (s == 0) return SZ_ERROR_ARCHIVE; \ pos += s; } @@ -1034,7 +1034,7 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, SRes res; ECoderFinishMode finishMode2 = finishMode; - BoolInt srcFinished2 = srcFinished; + BoolInt srcFinished2 = (BoolInt)srcFinished; BoolInt destFinish = False; if (p->block.packSize != (UInt64)(Int64)-1) @@ -1127,7 +1127,7 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, return SZ_OK; } - switch (p->state) + switch ((int)p->state) { case XZ_STATE_STREAM_HEADER: { @@ -1172,15 +1172,15 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, p->state = XZ_STATE_STREAM_INDEX; break; } - p->blockHeaderSize = ((UInt32)p->buf[0] << 2) + 4; + p->blockHeaderSize = ((unsigned)p->buf[0] << 2) + 4; break; } if (p->pos != p->blockHeaderSize) { - UInt32 cur = p->blockHeaderSize - p->pos; + unsigned cur = p->blockHeaderSize - p->pos; if (cur > srcRem) - cur = (UInt32)srcRem; + cur = (unsigned)srcRem; memcpy(p->buf + p->pos, src, cur); p->pos += cur; (*srcLen) += cur; @@ -1222,8 +1222,8 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, } else { - UInt32 checkSize = XzFlags_GetCheckSize(p->streamFlags); - UInt32 cur = checkSize - p->pos; + const unsigned checkSize = XzFlags_GetCheckSize(p->streamFlags); + unsigned cur = checkSize - p->pos; if (cur != 0) { if (srcRem == 0) @@ -1232,7 +1232,7 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, return SZ_OK; } if (cur > srcRem) - cur = (UInt32)srcRem; + cur = (unsigned)srcRem; memcpy(p->buf + p->pos, src, cur); p->pos += cur; (*srcLen) += cur; @@ -1321,9 +1321,9 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, case XZ_STATE_STREAM_FOOTER: { - UInt32 cur = XZ_STREAM_FOOTER_SIZE - p->pos; + unsigned cur = XZ_STREAM_FOOTER_SIZE - p->pos; if (cur > srcRem) - cur = (UInt32)srcRem; + cur = (unsigned)srcRem; memcpy(p->buf + p->pos, src, cur); p->pos += cur; (*srcLen) += cur; @@ -1358,6 +1358,8 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, } case XZ_STATE_BLOCK: break; /* to disable GCC warning */ + + default: return SZ_ERROR_FAIL; } } /* @@ -1773,10 +1775,10 @@ static void XzDecMt_Callback_Parse(void *obj, unsigned coderIndex, CMtDecCallbac } } { - UInt64 packSize = block->packSize; - UInt64 packSizeAligned = packSize + ((0 - (unsigned)packSize) & 3); - UInt32 checkSize = XzFlags_GetCheckSize(coder->dec.streamFlags); - UInt64 blockPackSum = coder->inPreSize + packSizeAligned + checkSize; + const UInt64 packSize = block->packSize; + const UInt64 packSizeAligned = packSize + ((0 - (unsigned)packSize) & 3); + const unsigned checkSize = XzFlags_GetCheckSize(coder->dec.streamFlags); + const UInt64 blockPackSum = coder->inPreSize + packSizeAligned + checkSize; // if (blockPackSum <= me->props.inBlockMax) // unpackBlockMaxSize { @@ -2381,7 +2383,7 @@ static SRes XzDecMt_Decode_ST(CXzDecMt *p if (tMode) { XzDecMt_FreeOutBufs(p); - tMode = MtDec_PrepareRead(&p->mtc); + tMode = (BoolInt)MtDec_PrepareRead(&p->mtc); } #endif @@ -2644,7 +2646,7 @@ SRes XzDecMt_Decode(CXzDecMtHandle p, p->outSize = *outDataSize; } - p->finishMode = finishMode; + p->finishMode = (BoolInt)finishMode; // p->outSize = 457; p->outSize_Defined = True; p->finishMode = False; // for test diff --git a/C/XzEnc.c b/C/XzEnc.c index 22408e2..c1affad 100644 --- a/C/XzEnc.c +++ b/C/XzEnc.c @@ -1,5 +1,5 @@ /* XzEnc.c -- Xz Encode -2023-04-13 : Igor Pavlov : Public domain */ +2024-03-01 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -29,8 +29,9 @@ #define XZ_GET_PAD_SIZE(dataSize) ((4 - ((unsigned)(dataSize) & 3)) & 3) -/* max pack size for LZMA2 block + check-64bytrs: */ -#define XZ_GET_MAX_BLOCK_PACK_SIZE(unpackSize) ((unpackSize) + ((unpackSize) >> 10) + 16 + 64) +#define XZ_CHECK_SIZE_MAX 64 +/* max pack size for LZMA2 block + pad4 + check_size: */ +#define XZ_GET_MAX_BLOCK_PACK_SIZE(unpackSize) ((unpackSize) + ((unpackSize) >> 10) + 16 + XZ_CHECK_SIZE_MAX) #define XZ_GET_ESTIMATED_BLOCK_TOTAL_PACK_SIZE(unpackSize) (XZ_BLOCK_HEADER_SIZE_MAX + XZ_GET_MAX_BLOCK_PACK_SIZE(unpackSize)) @@ -325,12 +326,13 @@ typedef struct static const z7_Func_BranchConv g_Funcs_BranchConv_RISC_Enc[] = { - Z7_BRANCH_CONV_ENC(PPC), - Z7_BRANCH_CONV_ENC(IA64), - Z7_BRANCH_CONV_ENC(ARM), - Z7_BRANCH_CONV_ENC(ARMT), - Z7_BRANCH_CONV_ENC(SPARC), - Z7_BRANCH_CONV_ENC(ARM64) + Z7_BRANCH_CONV_ENC_2 (BranchConv_PPC), + Z7_BRANCH_CONV_ENC_2 (BranchConv_IA64), + Z7_BRANCH_CONV_ENC_2 (BranchConv_ARM), + Z7_BRANCH_CONV_ENC_2 (BranchConv_ARMT), + Z7_BRANCH_CONV_ENC_2 (BranchConv_SPARC), + Z7_BRANCH_CONV_ENC_2 (BranchConv_ARM64), + Z7_BRANCH_CONV_ENC_2 (BranchConv_RISCV) }; static SizeT XzBcFilterStateBase_Filter_Enc(CXzBcFilterStateBase *p, Byte *data, SizeT size) @@ -888,9 +890,9 @@ static SRes Xz_CompressBlock( blockSizes->unpackSize = checkInStream.processed; } { - Byte buf[4 + 64]; - unsigned padSize = XZ_GET_PAD_SIZE(seqSizeOutStream.processed); - UInt64 packSize = seqSizeOutStream.processed; + Byte buf[4 + XZ_CHECK_SIZE_MAX]; + const unsigned padSize = XZ_GET_PAD_SIZE(seqSizeOutStream.processed); + const UInt64 packSize = seqSizeOutStream.processed; buf[0] = 0; buf[1] = 0; @@ -898,7 +900,8 @@ static SRes Xz_CompressBlock( buf[3] = 0; SeqCheckInStream_GetDigest(&checkInStream, buf + 4); - RINOK(WriteBytes(&seqSizeOutStream.vt, buf + (4 - padSize), padSize + XzFlags_GetCheckSize((CXzStreamFlags)props->checkId))) + RINOK(WriteBytes(&seqSizeOutStream.vt, buf + (4 - padSize), + padSize + XzFlags_GetCheckSize((CXzStreamFlags)props->checkId))) blockSizes->totalSize = seqSizeOutStream.processed - padSize; @@ -1083,18 +1086,19 @@ static SRes XzEnc_MtCallback_Code(void *pp, unsigned coderIndex, unsigned outBuf CXzEnc *me = (CXzEnc *)pp; SRes res; CMtProgressThunk progressThunk; - - Byte *dest = me->outBufs[outBufIndex]; - + Byte *dest; UNUSED_VAR(finished) - { CXzEncBlockInfo *bInfo = &me->EncBlocks[outBufIndex]; bInfo->totalSize = 0; bInfo->unpackSize = 0; bInfo->headerSize = 0; + // v23.02: we don't compress empty blocks + // also we must ignore that empty block in XzEnc_MtCallback_Write() + if (srcSize == 0) + return SZ_OK; } - + dest = me->outBufs[outBufIndex]; if (!dest) { dest = (Byte *)ISzAlloc_Alloc(me->alloc, me->outBufSize); @@ -1140,18 +1144,20 @@ static SRes XzEnc_MtCallback_Code(void *pp, unsigned coderIndex, unsigned outBuf static SRes XzEnc_MtCallback_Write(void *pp, unsigned outBufIndex) { CXzEnc *me = (CXzEnc *)pp; - const CXzEncBlockInfo *bInfo = &me->EncBlocks[outBufIndex]; - const Byte *data = me->outBufs[outBufIndex]; - - RINOK(WriteBytes(me->outStream, data, bInfo->headerSize)) - + // v23.02: we don't write empty blocks + // note: if (bInfo->unpackSize == 0) then there is no compressed data of block + if (bInfo->unpackSize == 0) + return SZ_OK; { - UInt64 totalPackFull = bInfo->totalSize + XZ_GET_PAD_SIZE(bInfo->totalSize); - RINOK(WriteBytes(me->outStream, data + XZ_BLOCK_HEADER_SIZE_MAX, (size_t)totalPackFull - bInfo->headerSize)) + const Byte *data = me->outBufs[outBufIndex]; + RINOK(WriteBytes(me->outStream, data, bInfo->headerSize)) + { + const UInt64 totalPackFull = bInfo->totalSize + XZ_GET_PAD_SIZE(bInfo->totalSize); + RINOK(WriteBytes(me->outStream, data + XZ_BLOCK_HEADER_SIZE_MAX, (size_t)totalPackFull - bInfo->headerSize)) + } + return XzEncIndex_AddIndexRecord(&me->xzIndex, bInfo->unpackSize, bInfo->totalSize, me->alloc); } - - return XzEncIndex_AddIndexRecord(&me->xzIndex, bInfo->unpackSize, bInfo->totalSize, me->alloc); } #endif diff --git a/C/XzIn.c b/C/XzIn.c index d0fc763..b68af96 100644 --- a/C/XzIn.c +++ b/C/XzIn.c @@ -1,5 +1,5 @@ /* XzIn.c - Xz input -2023-04-02 : Igor Pavlov : Public domain */ +2023-09-07 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -27,7 +27,7 @@ SRes Xz_ReadHeader(CXzStreamFlags *p, ISeqInStreamPtr inStream) } #define READ_VARINT_AND_CHECK(buf, pos, size, res) \ - { unsigned s = Xz_ReadVarInt(buf + pos, size - pos, res); \ + { const unsigned s = Xz_ReadVarInt(buf + pos, size - pos, res); \ if (s == 0) return SZ_ERROR_ARCHIVE; \ pos += s; } @@ -37,7 +37,7 @@ SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStreamPtr inStream, BoolInt *isIndex, unsigned headerSize; *headerSizeRes = 0; RINOK(SeqInStream_ReadByte(inStream, &header[0])) - headerSize = (unsigned)header[0]; + headerSize = header[0]; if (headerSize == 0) { *headerSizeRes = 1; @@ -47,7 +47,7 @@ SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStreamPtr inStream, BoolInt *isIndex, *isIndex = False; headerSize = (headerSize << 2) + 4; - *headerSizeRes = headerSize; + *headerSizeRes = (UInt32)headerSize; { size_t processedSize = headerSize - 1; RINOK(SeqInStream_ReadMax(inStream, header + 1, &processedSize)) @@ -58,7 +58,7 @@ SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStreamPtr inStream, BoolInt *isIndex, } #define ADD_SIZE_CHECK(size, val) \ - { UInt64 newSize = size + (val); if (newSize < size) return XZ_SIZE_OVERFLOW; size = newSize; } + { const UInt64 newSize = size + (val); if (newSize < size) return XZ_SIZE_OVERFLOW; size = newSize; } UInt64 Xz_GetUnpackSize(const CXzStream *p) { diff --git a/C/ZstdDec.c b/C/ZstdDec.c new file mode 100644 index 0000000..ecf6d22 --- /dev/null +++ b/C/ZstdDec.c @@ -0,0 +1,4064 @@ +/* ZstdDec.c -- Zstd Decoder +2024-01-21 : the code was developed by Igor Pavlov, using Zstandard format + specification and original zstd decoder code as reference code. +original zstd decoder code: Copyright (c) Facebook, Inc. All rights reserved. +This source code is licensed under BSD 3-Clause License. +*/ + +#include "Precomp.h" + +#include +#include +// #include + +#include "Alloc.h" +#include "Xxh64.h" +#include "ZstdDec.h" +#include "CpuArch.h" + +#if defined(MY_CPU_ARM64) +#include +#endif + +/* original-zstd still doesn't support window larger than 2 GiB. + So we also limit our decoder for 2 GiB window: */ +#if defined(MY_CPU_64BIT) && 0 == 1 + #define MAX_WINDOW_SIZE_LOG 41 +#else + #define MAX_WINDOW_SIZE_LOG 31 +#endif + +typedef + #if MAX_WINDOW_SIZE_LOG < 32 + UInt32 + #else + size_t + #endif + CZstdDecOffset; + +// for debug: simpler and smaller code but slow: +// #define Z7_ZSTD_DEC_USE_HUF_STREAM1_ALWAYS + +// #define SHOW_STAT +#ifdef SHOW_STAT +#include +static unsigned g_Num_Blocks_Compressed = 0; +static unsigned g_Num_Blocks_memcpy = 0; +static unsigned g_Num_Wrap_memmove_Num = 0; +static unsigned g_Num_Wrap_memmove_Bytes = 0; +static unsigned g_NumSeqs_total = 0; +// static unsigned g_NumCopy = 0; +static unsigned g_NumOver = 0; +static unsigned g_NumOver2 = 0; +static unsigned g_Num_Match = 0; +static unsigned g_Num_Lits = 0; +static unsigned g_Num_LitsBig = 0; +static unsigned g_Num_Lit0 = 0; +static unsigned g_Num_Rep0 = 0; +static unsigned g_Num_Rep1 = 0; +static unsigned g_Num_Rep2 = 0; +static unsigned g_Num_Rep3 = 0; +static unsigned g_Num_Threshold_0 = 0; +static unsigned g_Num_Threshold_1 = 0; +static unsigned g_Num_Threshold_0sum = 0; +static unsigned g_Num_Threshold_1sum = 0; +#define STAT_UPDATE(v) v +#else +#define STAT_UPDATE(v) +#endif +#define STAT_INC(v) STAT_UPDATE(v++;) + + +typedef struct +{ + const Byte *ptr; + size_t len; +} +CInBufPair; + + +#if defined(MY_CPU_ARM_OR_ARM64) || defined(MY_CPU_X86_OR_AMD64) + #if (defined(__clang__) && (__clang_major__ >= 6)) \ + || (defined(__GNUC__) && (__GNUC__ >= 6)) + // disable for debug: + #define Z7_ZSTD_DEC_USE_BSR + #elif defined(_MSC_VER) && (_MSC_VER >= 1300) + // #if defined(MY_CPU_ARM_OR_ARM64) + #if (_MSC_VER >= 1600) + #include + #endif + // disable for debug: + #define Z7_ZSTD_DEC_USE_BSR + #endif +#endif + +#ifdef Z7_ZSTD_DEC_USE_BSR + #if defined(__clang__) || defined(__GNUC__) + #define MY_clz(x) ((unsigned)__builtin_clz((UInt32)x)) + #else // #if defined(_MSC_VER) + #ifdef MY_CPU_ARM_OR_ARM64 + #define MY_clz _CountLeadingZeros + #endif // MY_CPU_X86_OR_AMD64 + #endif // _MSC_VER +#elif !defined(Z7_ZSTD_DEC_USE_LOG_TABLE) + #define Z7_ZSTD_DEC_USE_LOG_TABLE +#endif + + +static +Z7_FORCE_INLINE +unsigned GetHighestSetBit_32_nonzero_big(UInt32 num) +{ + // (num != 0) + #ifdef MY_clz + return 31 - MY_clz(num); + #elif defined(Z7_ZSTD_DEC_USE_BSR) + { + unsigned long zz; + _BitScanReverse(&zz, num); + return zz; + } + #else + { + int i = -1; + for (;;) + { + i++; + num >>= 1; + if (num == 0) + return (unsigned)i; + } + } + #endif +} + +#ifdef Z7_ZSTD_DEC_USE_LOG_TABLE + +#define R1(a) a, a +#define R2(a) R1(a), R1(a) +#define R3(a) R2(a), R2(a) +#define R4(a) R3(a), R3(a) +#define R5(a) R4(a), R4(a) +#define R6(a) R5(a), R5(a) +#define R7(a) R6(a), R6(a) +#define R8(a) R7(a), R7(a) +#define R9(a) R8(a), R8(a) + +#define Z7_ZSTD_FSE_MAX_ACCURACY 9 +// states[] values in FSE_Generate() can use (Z7_ZSTD_FSE_MAX_ACCURACY + 1) bits. +static const Byte k_zstd_LogTable[2 << Z7_ZSTD_FSE_MAX_ACCURACY] = +{ + R1(0), R1(1), R2(2), R3(3), R4(4), R5(5), R6(6), R7(7), R8(8), R9(9) +}; + +#define GetHighestSetBit_32_nonzero_small(num) (k_zstd_LogTable[num]) +#else +#define GetHighestSetBit_32_nonzero_small GetHighestSetBit_32_nonzero_big +#endif + + +#ifdef MY_clz + #define UPDATE_BIT_OFFSET_FOR_PADDING(b, bitOffset) \ + bitOffset -= (CBitCtr)(MY_clz(b) - 23); +#elif defined(Z7_ZSTD_DEC_USE_BSR) + #define UPDATE_BIT_OFFSET_FOR_PADDING(b, bitOffset) \ + { unsigned long zz; _BitScanReverse(&zz, b); bitOffset -= 8; bitOffset += zz; } +#else + #define UPDATE_BIT_OFFSET_FOR_PADDING(b, bitOffset) \ + for (;;) { bitOffset--; if (b & 0x80) { break; } b <<= 1; } +#endif + +#define SET_bitOffset_TO_PAD(bitOffset, src, srcLen) \ +{ \ + unsigned lastByte = (src)[(size_t)(srcLen) - 1]; \ + if (lastByte == 0) return SZ_ERROR_DATA; \ + bitOffset = (CBitCtr)((srcLen) * 8); \ + UPDATE_BIT_OFFSET_FOR_PADDING(lastByte, bitOffset) \ +} + +#ifndef Z7_ZSTD_DEC_USE_HUF_STREAM1_ALWAYS + +#define SET_bitOffset_TO_PAD_and_SET_BIT_SIZE(bitOffset, src, srcLen_res) \ +{ \ + unsigned lastByte = (src)[(size_t)(srcLen_res) - 1]; \ + if (lastByte == 0) return SZ_ERROR_DATA; \ + srcLen_res *= 8; \ + bitOffset = (CBitCtr)srcLen_res; \ + UPDATE_BIT_OFFSET_FOR_PADDING(lastByte, bitOffset) \ +} + +#endif + +/* +typedef Int32 CBitCtr_signed; +typedef Int32 CBitCtr; +*/ +// /* +typedef ptrdiff_t CBitCtr_signed; +typedef ptrdiff_t CBitCtr; +// */ + + +#define MATCH_LEN_MIN 3 +#define kBlockSizeMax (1u << 17) + +// #define Z7_ZSTD_DEC_PRINT_TABLE + +#ifdef Z7_ZSTD_DEC_PRINT_TABLE +#define NUM_OFFSET_SYMBOLS_PREDEF 29 +#endif +#define NUM_OFFSET_SYMBOLS_MAX (MAX_WINDOW_SIZE_LOG + 1) // 32 +#define NUM_LL_SYMBOLS 36 +#define NUM_ML_SYMBOLS 53 +#define FSE_NUM_SYMBOLS_MAX 53 // NUM_ML_SYMBOLS + +// /* +#if !defined(MY_CPU_X86) || defined(__PIC__) || defined(MY_CPU_64BIT) +#define Z7_ZSTD_DEC_USE_BASES_IN_OBJECT +#endif +// */ +// for debug: +// #define Z7_ZSTD_DEC_USE_BASES_LOCAL +// #define Z7_ZSTD_DEC_USE_BASES_IN_OBJECT + +#define GLOBAL_TABLE(n) k_ ## n + +#if defined(Z7_ZSTD_DEC_USE_BASES_LOCAL) + #define BASES_TABLE(n) a_ ## n +#elif defined(Z7_ZSTD_DEC_USE_BASES_IN_OBJECT) + #define BASES_TABLE(n) p->m_ ## n +#else + #define BASES_TABLE(n) GLOBAL_TABLE(n) +#endif + +#define Z7_ZSTD_DEC_USE_ML_PLUS3 + +#if defined(Z7_ZSTD_DEC_USE_BASES_LOCAL) || \ + defined(Z7_ZSTD_DEC_USE_BASES_IN_OBJECT) + +#define SEQ_EXTRA_TABLES(n) \ + Byte n ## SEQ_LL_EXTRA [NUM_LL_SYMBOLS]; \ + Byte n ## SEQ_ML_EXTRA [NUM_ML_SYMBOLS]; \ + UInt32 n ## SEQ_LL_BASES [NUM_LL_SYMBOLS]; \ + UInt32 n ## SEQ_ML_BASES [NUM_ML_SYMBOLS]; \ + +#define Z7_ZSTD_DEC_USE_BASES_CALC + +#ifdef Z7_ZSTD_DEC_USE_BASES_CALC + + #define FILL_LOC_BASES(n, startSum) \ + { unsigned i; UInt32 sum = startSum; \ + for (i = 0; i != Z7_ARRAY_SIZE(GLOBAL_TABLE(n ## _EXTRA)); i++) \ + { const unsigned a = GLOBAL_TABLE(n ## _EXTRA)[i]; \ + BASES_TABLE(n ## _BASES)[i] = sum; \ + /* if (sum != GLOBAL_TABLE(n ## _BASES)[i]) exit(1); */ \ + sum += (UInt32)1 << a; \ + BASES_TABLE(n ## _EXTRA)[i] = (Byte)a; }} + + #define FILL_LOC_BASES_ALL \ + FILL_LOC_BASES (SEQ_LL, 0) \ + FILL_LOC_BASES (SEQ_ML, MATCH_LEN_MIN) \ + +#else + #define COPY_GLOBAL_ARR(n) \ + memcpy(BASES_TABLE(n), GLOBAL_TABLE(n), sizeof(GLOBAL_TABLE(n))); + #define FILL_LOC_BASES_ALL \ + COPY_GLOBAL_ARR (SEQ_LL_EXTRA) \ + COPY_GLOBAL_ARR (SEQ_ML_EXTRA) \ + COPY_GLOBAL_ARR (SEQ_LL_BASES) \ + COPY_GLOBAL_ARR (SEQ_ML_BASES) \ + +#endif + +#endif + + + +/// The sequence decoding baseline and number of additional bits to read/add +#if !defined(Z7_ZSTD_DEC_USE_BASES_CALC) +static const UInt32 GLOBAL_TABLE(SEQ_LL_BASES) [NUM_LL_SYMBOLS] = +{ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 18, 20, 22, 24, 28, 32, 40, 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, + 0x2000, 0x4000, 0x8000, 0x10000 +}; +#endif + +static const Byte GLOBAL_TABLE(SEQ_LL_EXTRA) [NUM_LL_SYMBOLS] = +{ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9, 10, 11, 12, + 13, 14, 15, 16 +}; + +#if !defined(Z7_ZSTD_DEC_USE_BASES_CALC) +static const UInt32 GLOBAL_TABLE(SEQ_ML_BASES) [NUM_ML_SYMBOLS] = +{ + 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, + 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, + 35, 37, 39, 41, 43, 47, 51, 59, 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, + 0x1003, 0x2003, 0x4003, 0x8003, 0x10003 +}; +#endif + +static const Byte GLOBAL_TABLE(SEQ_ML_EXTRA) [NUM_ML_SYMBOLS] = +{ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 16 +}; + + +#ifdef Z7_ZSTD_DEC_PRINT_TABLE + +static const Int16 SEQ_LL_PREDEF_DIST [NUM_LL_SYMBOLS] = +{ + 4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1, + -1,-1,-1,-1 +}; +static const Int16 SEQ_OFFSET_PREDEF_DIST [NUM_OFFSET_SYMBOLS_PREDEF] = +{ + 1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1,-1,-1,-1,-1,-1 +}; +static const Int16 SEQ_ML_PREDEF_DIST [NUM_ML_SYMBOLS] = +{ + 1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,-1,-1, + -1,-1,-1,-1,-1 +}; + +#endif + +// typedef int FastInt; +// typedef Int32 FastInt32; +typedef unsigned FastInt; +typedef UInt32 FastInt32; +typedef FastInt32 CFseRecord; + + +#define FSE_REC_LEN_OFFSET 8 +#define FSE_REC_STATE_OFFSET 16 +#define GET_FSE_REC_SYM(st) ((Byte)(st)) +#define GET_FSE_REC_LEN(st) ((Byte)((st) >> FSE_REC_LEN_OFFSET)) +#define GET_FSE_REC_STATE(st) ((st) >> FSE_REC_STATE_OFFSET) + +// #define FSE_REC_SYM_MASK (0xff) +// #define GET_FSE_REC_SYM(st) (st & FSE_REC_SYM_MASK) + +#define W_BASE(state, len, sym) \ + (((UInt32)state << (4 + FSE_REC_STATE_OFFSET)) + \ + (len << FSE_REC_LEN_OFFSET) + (sym)) +#define W(state, len, sym) W_BASE(state, len, sym) +static const CFseRecord k_PredefRecords_LL[1 << 6] = { +W(0,4, 0),W(1,4, 0),W(2,5, 1),W(0,5, 3),W(0,5, 4),W(0,5, 6),W(0,5, 7),W(0,5, 9), +W(0,5,10),W(0,5,12),W(0,6,14),W(0,5,16),W(0,5,18),W(0,5,19),W(0,5,21),W(0,5,22), +W(0,5,24),W(2,5,25),W(0,5,26),W(0,6,27),W(0,6,29),W(0,6,31),W(2,4, 0),W(0,4, 1), +W(0,5, 2),W(2,5, 4),W(0,5, 5),W(2,5, 7),W(0,5, 8),W(2,5,10),W(0,5,11),W(0,6,13), +W(2,5,16),W(0,5,17),W(2,5,19),W(0,5,20),W(2,5,22),W(0,5,23),W(0,4,25),W(1,4,25), +W(2,5,26),W(0,6,28),W(0,6,30),W(3,4, 0),W(1,4, 1),W(2,5, 2),W(2,5, 3),W(2,5, 5), +W(2,5, 6),W(2,5, 8),W(2,5, 9),W(2,5,11),W(2,5,12),W(0,6,15),W(2,5,17),W(2,5,18), +W(2,5,20),W(2,5,21),W(2,5,23),W(2,5,24),W(0,6,35),W(0,6,34),W(0,6,33),W(0,6,32) +}; +static const CFseRecord k_PredefRecords_OF[1 << 5] = { +W(0,5, 0),W(0,4, 6),W(0,5, 9),W(0,5,15),W(0,5,21),W(0,5, 3),W(0,4, 7),W(0,5,12), +W(0,5,18),W(0,5,23),W(0,5, 5),W(0,4, 8),W(0,5,14),W(0,5,20),W(0,5, 2),W(1,4, 7), +W(0,5,11),W(0,5,17),W(0,5,22),W(0,5, 4),W(1,4, 8),W(0,5,13),W(0,5,19),W(0,5, 1), +W(1,4, 6),W(0,5,10),W(0,5,16),W(0,5,28),W(0,5,27),W(0,5,26),W(0,5,25),W(0,5,24) +}; +#if defined(Z7_ZSTD_DEC_USE_ML_PLUS3) +#undef W +#define W(state, len, sym) W_BASE(state, len, (sym + MATCH_LEN_MIN)) +#endif +static const CFseRecord k_PredefRecords_ML[1 << 6] = { +W(0,6, 0),W(0,4, 1),W(2,5, 2),W(0,5, 3),W(0,5, 5),W(0,5, 6),W(0,5, 8),W(0,6,10), +W(0,6,13),W(0,6,16),W(0,6,19),W(0,6,22),W(0,6,25),W(0,6,28),W(0,6,31),W(0,6,33), +W(0,6,35),W(0,6,37),W(0,6,39),W(0,6,41),W(0,6,43),W(0,6,45),W(1,4, 1),W(0,4, 2), +W(2,5, 3),W(0,5, 4),W(2,5, 6),W(0,5, 7),W(0,6, 9),W(0,6,12),W(0,6,15),W(0,6,18), +W(0,6,21),W(0,6,24),W(0,6,27),W(0,6,30),W(0,6,32),W(0,6,34),W(0,6,36),W(0,6,38), +W(0,6,40),W(0,6,42),W(0,6,44),W(2,4, 1),W(3,4, 1),W(1,4, 2),W(2,5, 4),W(2,5, 5), +W(2,5, 7),W(2,5, 8),W(0,6,11),W(0,6,14),W(0,6,17),W(0,6,20),W(0,6,23),W(0,6,26), +W(0,6,29),W(0,6,52),W(0,6,51),W(0,6,50),W(0,6,49),W(0,6,48),W(0,6,47),W(0,6,46) +}; + + +// sum of freqs[] must be correct +// (numSyms != 0) +// (accuracy >= 5) +static +Z7_NO_INLINE +// Z7_FORCE_INLINE +void FSE_Generate(CFseRecord *table, + const Int16 *const freqs, const size_t numSyms, + const unsigned accuracy, UInt32 delta) +{ + size_t size = (size_t)1 << accuracy; + // max value in states[x] is ((1 << accuracy) * 2) + UInt16 states[FSE_NUM_SYMBOLS_MAX]; + { + /* Symbols with "less than 1" probability get a single cell, + starting from the end of the table. + These symbols define a full state reset, reading (accuracy) bits. */ + size_t threshold = size; + { + size_t s = 0; + do + if (freqs[s] == -1) + { + table[--threshold] = (CFseRecord)s; + states[s] = 1; + } + while (++s != numSyms); + } + + #ifdef SHOW_STAT + if (threshold == size) + { + STAT_INC(g_Num_Threshold_0) + STAT_UPDATE(g_Num_Threshold_0sum += (unsigned)size;) + } + else + { + STAT_INC(g_Num_Threshold_1) + STAT_UPDATE(g_Num_Threshold_1sum += (unsigned)size;) + } + #endif + + // { unsigned uuu; for (uuu = 0; uuu < 400; uuu++) + { + // Each (symbol) gets freqs[symbol] cells. + // Cell allocation is spread, not linear. + const size_t step = (size >> 1) + (size >> 3) + 3; + size_t pos = 0; + // const unsigned mask = size - 1; + /* + if (threshold == size) + { + size_t s = 0; + size--; + do + { + int freq = freqs[s]; + if (freq <= 0) + continue; + states[s] = (UInt16)freq; + do + { + table[pos] (CFseRecord)s; + pos = (pos + step) & size; // & mask; + } + while (--freq); + } + while (++s != numSyms); + } + else + */ + { + size_t s = 0; + size--; + do + { + int freq = freqs[s]; + if (freq <= 0) + continue; + states[s] = (UInt16)freq; + do + { + table[pos] = (CFseRecord)s; + // we skip position, if it's already occupied by a "less than 1" probability symbol. + // (step) is coprime to table size, so the cycle will visit each position exactly once + do + pos = (pos + step) & size; // & mask; + while (pos >= threshold); + } + while (--freq); + } + while (++s != numSyms); + } + size++; + // (pos != 0) is unexpected case that means that freqs[] are not correct. + // so it's some failure in code (for example, incorrect predefined freq[] table) + // if (pos != 0) return SZ_ERROR_FAIL; + } + // } + } + { + const CFseRecord * const limit = table + size; + delta = ((UInt32)size << FSE_REC_STATE_OFFSET) - delta; + /* State increases by symbol over time, decreasing number of bits. + Baseline increases until the bit threshold is passed, at which point it resets to 0 */ + do + { + #define TABLE_ITER(a) \ + { \ + const FastInt sym = (FastInt)table[a]; \ + const unsigned nextState = states[sym]; \ + unsigned nb; \ + states[sym] = (UInt16)(nextState + 1); \ + nb = accuracy - GetHighestSetBit_32_nonzero_small(nextState); \ + table[a] = (CFseRecord)(sym - delta \ + + ((UInt32)nb << FSE_REC_LEN_OFFSET) \ + + ((UInt32)nextState << FSE_REC_STATE_OFFSET << nb)); \ + } + TABLE_ITER(0) + TABLE_ITER(1) + table += 2; + } + while (table != limit); + } +} + + +#ifdef Z7_ZSTD_DEC_PRINT_TABLE + +static void Print_Predef(unsigned predefAccuracy, + const unsigned numSymsPredef, + const Int16 * const predefFreqs, + const CFseRecord *checkTable) +{ + CFseRecord table[1 << 6]; + unsigned i; + FSE_Generate(table, predefFreqs, numSymsPredef, predefAccuracy, + #if defined(Z7_ZSTD_DEC_USE_ML_PLUS3) + numSymsPredef == NUM_ML_SYMBOLS ? MATCH_LEN_MIN : + #endif + 0 + ); + if (memcmp(table, checkTable, sizeof(UInt32) << predefAccuracy) != 0) + exit(1); + for (i = 0; i < (1u << predefAccuracy); i++) + { + const UInt32 v = table[i]; + const unsigned state = (unsigned)(GET_FSE_REC_STATE(v)); + if (state & 0xf) + exit(1); + if (i != 0) + { + printf(","); + if (i % 8 == 0) + printf("\n"); + } + printf("W(%d,%d,%2d)", + (unsigned)(state >> 4), + (unsigned)((v >> FSE_REC_LEN_OFFSET) & 0xff), + (unsigned)GET_FSE_REC_SYM(v)); + } + printf("\n\n"); +} + +#endif + + +#define GET16(dest, p) { const Byte *ptr = p; dest = GetUi16(ptr); } +#define GET32(dest, p) { const Byte *ptr = p; dest = GetUi32(ptr); } + +// (1 <= numBits <= 9) +#define FORWARD_READ_BITS(destVal, numBits, mask) \ + { const CBitCtr_signed bos3 = (bitOffset) >> 3; \ + if (bos3 >= 0) return SZ_ERROR_DATA; \ + GET16(destVal, src + bos3) \ + destVal >>= (bitOffset) & 7; \ + bitOffset += (CBitCtr_signed)(numBits); \ + mask = (1u << (numBits)) - 1; \ + destVal &= mask; \ + } + +#define FORWARD_READ_1BIT(destVal) \ + { const CBitCtr_signed bos3 = (bitOffset) >> 3; \ + if (bos3 >= 0) return SZ_ERROR_DATA; \ + destVal = *(src + bos3); \ + destVal >>= (bitOffset) & 7; \ + (bitOffset)++; \ + destVal &= 1; \ + } + + +// in: (accuracyMax <= 9) +// at least 2 bytes will be processed from (in) stream. +// at return: (in->len > 0) +static +Z7_NO_INLINE +SRes FSE_DecodeHeader(CFseRecord *const table, + CInBufPair *const in, + const unsigned accuracyMax, + Byte *const accuracyRes, + unsigned numSymbolsMax) +{ + unsigned accuracy; + unsigned remain1; + unsigned syms; + Int16 freqs[FSE_NUM_SYMBOLS_MAX + 3]; // +3 for overwrite (repeat) + const Byte *src = in->ptr; + CBitCtr_signed bitOffset = (CBitCtr_signed)in->len - 1; + if (bitOffset <= 0) + return SZ_ERROR_DATA; + accuracy = *src & 0xf; + accuracy += 5; + if (accuracy > accuracyMax) + return SZ_ERROR_DATA; + *accuracyRes = (Byte)accuracy; + remain1 = (1u << accuracy) + 1; // (it's remain_freqs_sum + 1) + syms = 0; + src += bitOffset; // src points to last byte + bitOffset = 4 - (bitOffset << 3); + + for (;;) + { + // (2 <= remain1) + const unsigned bits = GetHighestSetBit_32_nonzero_small((unsigned)remain1); + // (1 <= bits <= accuracy) + unsigned val; // it must be unsigned or int + unsigned mask; + FORWARD_READ_BITS(val, bits, mask) + { + const unsigned val2 = remain1 + val - mask; + if (val2 > mask) + { + unsigned bit; + FORWARD_READ_1BIT(bit) + if (bit) + val = val2; + } + } + { + // (remain1 >= 2) + // (0 <= (int)val <= remain1) + val = (unsigned)((int)val - 1); + // val now is "probability" of symbol + // (probability == -1) means "less than 1" frequency. + // (-1 <= (int)val <= remain1 - 1) + freqs[syms++] = (Int16)(int)val; + if (val != 0) + { + remain1 -= (int)val < 0 ? 1u : (unsigned)val; + // remain1 -= val; + // val >>= (sizeof(val) * 8 - 2); + // remain1 -= val & 2; + // freqs[syms++] = (Int16)(int)val; + // syms++; + if (remain1 == 1) + break; + if (syms >= FSE_NUM_SYMBOLS_MAX) + return SZ_ERROR_DATA; + } + else // if (val == 0) + { + // freqs[syms++] = 0; + // syms++; + for (;;) + { + unsigned repeat; + FORWARD_READ_BITS(repeat, 2, mask) + freqs[syms ] = 0; + freqs[syms + 1] = 0; + freqs[syms + 2] = 0; + syms += repeat; + if (syms >= FSE_NUM_SYMBOLS_MAX) + return SZ_ERROR_DATA; + if (repeat != 3) + break; + } + } + } + } + + if (syms > numSymbolsMax) + return SZ_ERROR_DATA; + bitOffset += 7; + bitOffset >>= 3; + if (bitOffset > 0) + return SZ_ERROR_DATA; + in->ptr = src + bitOffset; + in->len = (size_t)(1 - bitOffset); + { + // unsigned uuu; for (uuu = 0; uuu < 50; uuu++) + FSE_Generate(table, freqs, syms, accuracy, + #if defined(Z7_ZSTD_DEC_USE_ML_PLUS3) + numSymbolsMax == NUM_ML_SYMBOLS ? MATCH_LEN_MIN : + #endif + 0 + ); + } + return SZ_OK; +} + + +// ---------- HUFFMAN ---------- + +#define HUF_MAX_BITS 12 +#define HUF_MAX_SYMBS 256 +#define HUF_DUMMY_SIZE (128 + 8 * 2) // it must multiple of 8 +// #define HUF_DUMMY_SIZE 0 +#define HUF_TABLE_SIZE ((2 << HUF_MAX_BITS) + HUF_DUMMY_SIZE) +#define HUF_GET_SYMBOLS(table) ((table) + (1 << HUF_MAX_BITS) + HUF_DUMMY_SIZE) +// #define HUF_GET_LENS(table) (table) + +typedef struct +{ + // Byte table[HUF_TABLE_SIZE]; + UInt64 table64[HUF_TABLE_SIZE / sizeof(UInt64)]; +} +CZstdDecHufTable; + +/* +Input: + numSyms != 0 + (bits) array size must be aligned for 2 + if (numSyms & 1), then bits[numSyms] == 0, + Huffman tree must be correct before Huf_Build() call: + (sum (1/2^bits[i]) == 1). + && (bits[i] <= HUF_MAX_BITS) +*/ +static +Z7_FORCE_INLINE +void Huf_Build(Byte * const table, + const Byte *bits, const unsigned numSyms) +{ + unsigned counts0[HUF_MAX_BITS + 1]; + unsigned counts1[HUF_MAX_BITS + 1]; + const Byte * const bitsEnd = bits + numSyms; + // /* + { + unsigned t; + for (t = 0; t < Z7_ARRAY_SIZE(counts0); t++) counts0[t] = 0; + for (t = 0; t < Z7_ARRAY_SIZE(counts1); t++) counts1[t] = 0; + } + // */ + // memset(counts0, 0, sizeof(counts0)); + // memset(counts1, 0, sizeof(counts1)); + { + const Byte *bits2 = bits; + // we access additional bits[symbol] if (numSyms & 1) + do + { + counts0[bits2[0]]++; + counts1[bits2[1]]++; + } + while ((bits2 += 2) < bitsEnd); + } + { + unsigned r = 0; + unsigned i = HUF_MAX_BITS; + // Byte *lens = HUF_GET_LENS(symbols); + do + { + const unsigned num = (counts0[i] + counts1[i]) << (HUF_MAX_BITS - i); + counts0[i] = r; + if (num) + { + Byte *lens = &table[r]; + r += num; + memset(lens, (int)i, num); + } + } + while (--i); + counts0[0] = 0; // for speculated loads + // no need for check: + // if (r != (UInt32)1 << HUF_MAX_BITS) exit(0); + } + { + #ifdef MY_CPU_64BIT + UInt64 + #else + UInt32 + #endif + v = 0; + Byte *symbols = HUF_GET_SYMBOLS(table); + do + { + const unsigned nb = *bits++; + if (nb) + { + const unsigned code = counts0[nb]; + const unsigned num = (1u << HUF_MAX_BITS) >> nb; + counts0[nb] = code + num; + // memset(&symbols[code], i, num); + // /* + { + Byte *s2 = &symbols[code]; + if (num <= 2) + { + s2[0] = (Byte)v; + s2[(size_t)num - 1] = (Byte)v; + } + else if (num <= 8) + { + *(UInt32 *)(void *)s2 = (UInt32)v; + *(UInt32 *)(void *)(s2 + (size_t)num - 4) = (UInt32)v; + } + else + { + #ifdef MY_CPU_64BIT + UInt64 *s = (UInt64 *)(void *)s2; + const UInt64 *lim = (UInt64 *)(void *)(s2 + num); + do + { + s[0] = v; s[1] = v; s += 2; + } + while (s != lim); + #else + UInt32 *s = (UInt32 *)(void *)s2; + const UInt32 *lim = (const UInt32 *)(const void *)(s2 + num); + do + { + s[0] = v; s[1] = v; s += 2; + s[0] = v; s[1] = v; s += 2; + } + while (s != lim); + #endif + } + } + // */ + } + v += + #ifdef MY_CPU_64BIT + 0x0101010101010101; + #else + 0x01010101; + #endif + } + while (bits != bitsEnd); + } +} + + + +// how many bytes (src) was moved back from original value. +// we need (HUF_SRC_OFFSET == 3) for optimized 32-bit memory access +#define HUF_SRC_OFFSET 3 + +// v <<= 8 - (bitOffset & 7) + numBits; +// v >>= 32 - HUF_MAX_BITS; +#define HUF_GET_STATE(v, bitOffset, numBits) \ + GET32(v, src + (HUF_SRC_OFFSET - 3) + ((CBitCtr_signed)bitOffset >> 3)) \ + v >>= 32 - HUF_MAX_BITS - 8 + ((unsigned)bitOffset & 7) - numBits; \ + v &= (1u << HUF_MAX_BITS) - 1; \ + + +#ifndef Z7_ZSTD_DEC_USE_HUF_STREAM1_ALWAYS +#if defined(MY_CPU_AMD64) && defined(_MSC_VER) && _MSC_VER == 1400 \ + || !defined(MY_CPU_X86_OR_AMD64) \ + // || 1 == 1 /* for debug : to force STREAM4_PRELOAD mode */ + // we need big number (>=16) of registers for PRELOAD4 + #define Z7_ZSTD_DEC_USE_HUF_STREAM4_PRELOAD4 + // #define Z7_ZSTD_DEC_USE_HUF_STREAM4_PRELOAD2 // for debug +#endif +#endif + +// for debug: simpler and smaller code but slow: +// #define Z7_ZSTD_DEC_USE_HUF_STREAM1_SIMPLE + +#if defined(Z7_ZSTD_DEC_USE_HUF_STREAM1_SIMPLE) || \ + !defined(Z7_ZSTD_DEC_USE_HUF_STREAM1_ALWAYS) + +#define HUF_DECODE(bitOffset, dest) \ +{ \ + UInt32 v; \ + HUF_GET_STATE(v, bitOffset, 0) \ + bitOffset -= table[v]; \ + *(dest) = symbols[v]; \ + if ((CBitCtr_signed)bitOffset < 0) return SZ_ERROR_DATA; \ +} + +#endif + +#if !defined(Z7_ZSTD_DEC_USE_HUF_STREAM1_SIMPLE) || \ + defined(Z7_ZSTD_DEC_USE_HUF_STREAM4_PRELOAD4) || \ + defined(Z7_ZSTD_DEC_USE_HUF_STREAM4_PRELOAD2) \ + +#define HUF_DECODE_2_INIT(v, bitOffset) \ + HUF_GET_STATE(v, bitOffset, 0) + +#define HUF_DECODE_2(v, bitOffset, dest) \ +{ \ + unsigned numBits; \ + numBits = table[v]; \ + *(dest) = symbols[v]; \ + HUF_GET_STATE(v, bitOffset, numBits) \ + bitOffset -= (CBitCtr)numBits; \ + if ((CBitCtr_signed)bitOffset < 0) return SZ_ERROR_DATA; \ +} + +#endif + + +// src == ptr - HUF_SRC_OFFSET +// we are allowed to access 3 bytes before start of input buffer +static +Z7_NO_INLINE +SRes Huf_Decompress_1stream(const Byte * const table, + const Byte *src, const size_t srcLen, + Byte *dest, const size_t destLen) +{ + CBitCtr bitOffset; + if (srcLen == 0) + return SZ_ERROR_DATA; + SET_bitOffset_TO_PAD (bitOffset, src + HUF_SRC_OFFSET, srcLen) + if (destLen) + { + const Byte *symbols = HUF_GET_SYMBOLS(table); + const Byte *destLim = dest + destLen; + #ifdef Z7_ZSTD_DEC_USE_HUF_STREAM1_SIMPLE + { + do + { + HUF_DECODE (bitOffset, dest) + } + while (++dest != destLim); + } + #else + { + UInt32 v; + HUF_DECODE_2_INIT (v, bitOffset) + do + { + HUF_DECODE_2 (v, bitOffset, dest) + } + while (++dest != destLim); + } + #endif + } + return bitOffset == 0 ? SZ_OK : SZ_ERROR_DATA; +} + + +// for debug : it reduces register pressure : by array copy can be slow : +// #define Z7_ZSTD_DEC_USE_HUF_LOCAL + +// src == ptr + (6 - HUF_SRC_OFFSET) +// srcLen >= 10 +// we are allowed to access 3 bytes before start of input buffer +static +Z7_NO_INLINE +SRes Huf_Decompress_4stream(const Byte * const + #ifdef Z7_ZSTD_DEC_USE_HUF_LOCAL + table2, + #else + table, + #endif + const Byte *src, size_t srcLen, + Byte *dest, size_t destLen) +{ + #ifdef Z7_ZSTD_DEC_USE_HUF_LOCAL + Byte table[HUF_TABLE_SIZE]; + #endif + UInt32 sizes[3]; + const size_t delta = (destLen + 3) / 4; + if ((sizes[0] = GetUi16(src + (0 + HUF_SRC_OFFSET - 6))) == 0) return SZ_ERROR_DATA; + if ((sizes[1] = GetUi16(src + (2 + HUF_SRC_OFFSET - 6))) == 0) return SZ_ERROR_DATA; + sizes[1] += sizes[0]; + if ((sizes[2] = GetUi16(src + (4 + HUF_SRC_OFFSET - 6))) == 0) return SZ_ERROR_DATA; + sizes[2] += sizes[1]; + srcLen -= 6; + if (srcLen <= sizes[2]) + return SZ_ERROR_DATA; + + #ifdef Z7_ZSTD_DEC_USE_HUF_LOCAL + { + // unsigned i = 0; for(; i < 1000; i++) + memcpy(table, table2, HUF_TABLE_SIZE); + } + #endif + + #ifndef Z7_ZSTD_DEC_USE_HUF_STREAM1_ALWAYS + { + CBitCtr bitOffset_0, + bitOffset_1, + bitOffset_2, + bitOffset_3; + { + SET_bitOffset_TO_PAD_and_SET_BIT_SIZE (bitOffset_0, src + HUF_SRC_OFFSET, sizes[0]) + SET_bitOffset_TO_PAD_and_SET_BIT_SIZE (bitOffset_1, src + HUF_SRC_OFFSET, sizes[1]) + SET_bitOffset_TO_PAD_and_SET_BIT_SIZE (bitOffset_2, src + HUF_SRC_OFFSET, sizes[2]) + SET_bitOffset_TO_PAD (bitOffset_3, src + HUF_SRC_OFFSET, srcLen) + } + { + const Byte * const symbols = HUF_GET_SYMBOLS(table); + Byte *destLim = dest + destLen - delta * 3; + + if (dest != destLim) + #ifdef Z7_ZSTD_DEC_USE_HUF_STREAM4_PRELOAD4 + { + UInt32 v_0, v_1, v_2, v_3; + HUF_DECODE_2_INIT (v_0, bitOffset_0) + HUF_DECODE_2_INIT (v_1, bitOffset_1) + HUF_DECODE_2_INIT (v_2, bitOffset_2) + HUF_DECODE_2_INIT (v_3, bitOffset_3) + // #define HUF_DELTA (1 << 17) / 4 + do + { + HUF_DECODE_2 (v_3, bitOffset_3, dest + delta * 3) + HUF_DECODE_2 (v_2, bitOffset_2, dest + delta * 2) + HUF_DECODE_2 (v_1, bitOffset_1, dest + delta) + HUF_DECODE_2 (v_0, bitOffset_0, dest) + } + while (++dest != destLim); + /* + {// unsigned y = 0; for (;y < 1; y++) + { + const size_t num = destLen - delta * 3; + Byte *orig = dest - num; + memmove (orig + delta , orig + HUF_DELTA, num); + memmove (orig + delta * 2, orig + HUF_DELTA * 2, num); + memmove (orig + delta * 3, orig + HUF_DELTA * 3, num); + }} + */ + } + #elif defined(Z7_ZSTD_DEC_USE_HUF_STREAM4_PRELOAD2) + { + UInt32 v_0, v_1, v_2, v_3; + HUF_DECODE_2_INIT (v_0, bitOffset_0) + HUF_DECODE_2_INIT (v_1, bitOffset_1) + do + { + HUF_DECODE_2 (v_0, bitOffset_0, dest) + HUF_DECODE_2 (v_1, bitOffset_1, dest + delta) + } + while (++dest != destLim); + dest = destLim - (destLen - delta * 3); + dest += delta * 2; + destLim += delta * 2; + HUF_DECODE_2_INIT (v_2, bitOffset_2) + HUF_DECODE_2_INIT (v_3, bitOffset_3) + do + { + HUF_DECODE_2 (v_2, bitOffset_2, dest) + HUF_DECODE_2 (v_3, bitOffset_3, dest + delta) + } + while (++dest != destLim); + dest -= delta * 2; + destLim -= delta * 2; + } + #else + { + do + { + HUF_DECODE (bitOffset_3, dest + delta * 3) + HUF_DECODE (bitOffset_2, dest + delta * 2) + HUF_DECODE (bitOffset_1, dest + delta) + HUF_DECODE (bitOffset_0, dest) + } + while (++dest != destLim); + } + #endif + + if (bitOffset_3 != (CBitCtr)sizes[2]) + return SZ_ERROR_DATA; + if (destLen &= 3) + { + destLim = dest + 4 - destLen; + do + { + HUF_DECODE (bitOffset_2, dest + delta * 2) + HUF_DECODE (bitOffset_1, dest + delta) + HUF_DECODE (bitOffset_0, dest) + } + while (++dest != destLim); + } + if ( bitOffset_0 != 0 + || bitOffset_1 != (CBitCtr)sizes[0] + || bitOffset_2 != (CBitCtr)sizes[1]) + return SZ_ERROR_DATA; + } + } + #else // Z7_ZSTD_DEC_USE_HUF_STREAM1_ALWAYS + { + unsigned i; + for (i = 0; i < 4; i++) + { + size_t d = destLen; + size_t size = srcLen; + if (i != 3) + { + d = delta; + size = sizes[i]; + } + if (i != 0) + size -= sizes[i - 1]; + destLen -= d; + RINOK(Huf_Decompress_1stream(table, src, size, dest, d)) + dest += d; + src += size; + } + } + #endif + + return SZ_OK; +} + + + +// (in->len != 0) +// we are allowed to access in->ptr[-3] +// at least 2 bytes in (in->ptr) will be processed +static SRes Huf_DecodeTable(CZstdDecHufTable *const p, CInBufPair *const in) +{ + Byte weights[HUF_MAX_SYMBS + 1]; // +1 for extra write for loop unroll + unsigned numSyms; + const unsigned header = *(in->ptr)++; + in->len--; + // memset(weights, 0, sizeof(weights)); + if (header >= 128) + { + // direct representation: 4 bits field (0-15) per weight + numSyms = header - 127; + // numSyms != 0 + { + const size_t numBytes = (numSyms + 1) / 2; + const Byte *const ws = in->ptr; + size_t i = 0; + if (in->len < numBytes) + return SZ_ERROR_DATA; + in->ptr += numBytes; + in->len -= numBytes; + do + { + const unsigned b = ws[i]; + weights[i * 2 ] = (Byte)(b >> 4); + weights[i * 2 + 1] = (Byte)(b & 0xf); + } + while (++i != numBytes); + /* 7ZIP: we can restore correct zero value for weights[numSyms], + if we want to use zero values starting from numSyms in code below. */ + // weights[numSyms] = 0; + } + } + else + { + #define MAX_ACCURACY_LOG_FOR_WEIGHTS 6 + CFseRecord table[1 << MAX_ACCURACY_LOG_FOR_WEIGHTS]; + + Byte accuracy; + const Byte *src; + size_t srcLen; + if (in->len < header) + return SZ_ERROR_DATA; + { + CInBufPair fse_stream; + fse_stream.len = header; + fse_stream.ptr = in->ptr; + in->ptr += header; + in->len -= header; + RINOK(FSE_DecodeHeader(table, &fse_stream, + MAX_ACCURACY_LOG_FOR_WEIGHTS, + &accuracy, + 16 // num weight symbols max (max-symbol is 15) + )) + // at least 2 bytes were processed in fse_stream. + // (srcLen > 0) after FSE_DecodeHeader() + // if (srcLen == 0) return SZ_ERROR_DATA; + src = fse_stream.ptr; + srcLen = fse_stream.len; + } + // we are allowed to access src[-5] + { + // unsigned yyy = 200; do { + CBitCtr bitOffset; + FastInt32 state1, state2; + SET_bitOffset_TO_PAD (bitOffset, src, srcLen) + state1 = accuracy; + src -= state1 >> 2; // src -= 1; // for GET16() optimization + state1 <<= FSE_REC_LEN_OFFSET; + state2 = state1; + numSyms = 0; + for (;;) + { + #define FSE_WEIGHT_DECODE(st) \ + { \ + const unsigned bits = GET_FSE_REC_LEN(st); \ + FastInt r; \ + GET16(r, src + (bitOffset >> 3)) \ + r >>= (unsigned)bitOffset & 7; \ + if ((CBitCtr_signed)(bitOffset -= (CBitCtr)bits) < 0) \ + { if (bitOffset + (CBitCtr)bits != 0) \ + return SZ_ERROR_DATA; \ + break; } \ + r &= 0xff; \ + r >>= 8 - bits; \ + st = table[GET_FSE_REC_STATE(st) + r]; \ + weights[numSyms++] = (Byte)GET_FSE_REC_SYM(st); \ + } + FSE_WEIGHT_DECODE (state1) + FSE_WEIGHT_DECODE (state2) + if (numSyms == HUF_MAX_SYMBS) + return SZ_ERROR_DATA; + } + // src += (unsigned)accuracy >> 2; } while (--yyy); + } + } + + // Build using weights: + { + UInt32 sum = 0; + { + // numSyms >= 1 + unsigned i = 0; + weights[numSyms] = 0; + do + { + sum += ((UInt32)1 << weights[i ]) & ~(UInt32)1; + sum += ((UInt32)1 << weights[i + 1]) & ~(UInt32)1; + i += 2; + } + while (i < numSyms); + if (sum == 0) + return SZ_ERROR_DATA; + } + { + const unsigned maxBits = GetHighestSetBit_32_nonzero_big(sum) + 1; + { + const UInt32 left = ((UInt32)1 << maxBits) - sum; + // (left != 0) + // (left) must be power of 2 in correct stream + if (left & (left - 1)) + return SZ_ERROR_DATA; + weights[numSyms++] = (Byte)GetHighestSetBit_32_nonzero_big(left); + } + // if (numSyms & 1) + weights[numSyms] = 0; // for loop unroll + // numSyms >= 2 + { + unsigned i = 0; + do + { + /* + #define WEIGHT_ITER(a) \ + { unsigned w = weights[i + (a)]; \ + const unsigned t = maxBits - w; \ + w = w ? t: w; \ + if (w > HUF_MAX_BITS) return SZ_ERROR_DATA; \ + weights[i + (a)] = (Byte)w; } + */ + // /* + #define WEIGHT_ITER(a) \ + { unsigned w = weights[i + (a)]; \ + if (w) { \ + w = maxBits - w; \ + if (w > HUF_MAX_BITS) return SZ_ERROR_DATA; \ + weights[i + (a)] = (Byte)w; }} + // */ + WEIGHT_ITER(0) + // WEIGHT_ITER(1) + // i += 2; + } + while (++i != numSyms); + } + } + } + { + // unsigned yyy; for (yyy = 0; yyy < 100; yyy++) + Huf_Build((Byte *)(void *)p->table64, weights, numSyms); + } + return SZ_OK; +} + + +typedef enum +{ + k_SeqMode_Predef = 0, + k_SeqMode_RLE = 1, + k_SeqMode_FSE = 2, + k_SeqMode_Repeat = 3 +} +z7_zstd_enum_SeqMode; + +// predefAccuracy == 5 for OFFSET symbols +// predefAccuracy == 6 for MATCH/LIT LEN symbols +static +SRes +Z7_NO_INLINE +// Z7_FORCE_INLINE +FSE_Decode_SeqTable(CFseRecord * const table, + CInBufPair * const in, + unsigned predefAccuracy, + Byte * const accuracyRes, + unsigned numSymbolsMax, + const CFseRecord * const predefs, + const unsigned seqMode) +{ + // UNUSED_VAR(numSymsPredef) + // UNUSED_VAR(predefFreqs) + if (seqMode == k_SeqMode_FSE) + { + // unsigned y = 50; CInBufPair in2 = *in; do { *in = in2; RINOK( + return + FSE_DecodeHeader(table, in, + predefAccuracy + 3, // accuracyMax + accuracyRes, + numSymbolsMax) + ; + // )} while (--y); return SZ_OK; + } + // numSymsMax = numSymsPredef + ((predefAccuracy & 1) * (32 - 29))); // numSymsMax + // numSymsMax == 32 for offsets + + if (seqMode == k_SeqMode_Predef) + { + *accuracyRes = (Byte)predefAccuracy; + memcpy(table, predefs, sizeof(UInt32) << predefAccuracy); + return SZ_OK; + } + + // (seqMode == k_SeqMode_RLE) + if (in->len == 0) + return SZ_ERROR_DATA; + in->len--; + { + const Byte *ptr = in->ptr; + const Byte sym = ptr[0]; + in->ptr = ptr + 1; + table[0] = (FastInt32)sym + #if defined(Z7_ZSTD_DEC_USE_ML_PLUS3) + + (numSymbolsMax == NUM_ML_SYMBOLS ? MATCH_LEN_MIN : 0) + #endif + ; + *accuracyRes = 0; + } + return SZ_OK; +} + + +typedef struct +{ + CFseRecord of[1 << 8]; + CFseRecord ll[1 << 9]; + CFseRecord ml[1 << 9]; +} +CZstdDecFseTables; + + +typedef struct +{ + Byte *win; + SizeT cycSize; + /* + if (outBuf_fromCaller) : cycSize = outBufSize_fromCaller + else { + if ( isCyclicMode) : cycSize = cyclic_buffer_size = (winSize + extra_space) + if (!isCyclicMode) : cycSize = ContentSize, + (isCyclicMode == true) if (ContetSize >= winSize) or ContetSize is unknown + } + */ + SizeT winPos; + + CZstdDecOffset reps[3]; + + Byte ll_accuracy; + Byte of_accuracy; + Byte ml_accuracy; + // Byte seqTables_wereSet; + Byte litHuf_wasSet; + + Byte *literalsBase; + + size_t winSize; // from header + size_t totalOutCheck; // totalOutCheck <= winSize + + #ifdef Z7_ZSTD_DEC_USE_BASES_IN_OBJECT + SEQ_EXTRA_TABLES(m_) + #endif + // UInt64 _pad_Alignment; // is not required now + CZstdDecFseTables fse; + CZstdDecHufTable huf; +} +CZstdDec1; + +#define ZstdDec1_GET_BLOCK_SIZE_LIMIT(p) \ + ((p)->winSize < kBlockSizeMax ? (UInt32)(p)->winSize : kBlockSizeMax) + +#define SEQ_TABLES_WERE_NOT_SET_ml_accuracy 1 // accuracy=1 is not used by zstd +#define IS_SEQ_TABLES_WERE_SET(p) (((p)->ml_accuracy != SEQ_TABLES_WERE_NOT_SET_ml_accuracy)) +// #define IS_SEQ_TABLES_WERE_SET(p) ((p)->seqTables_wereSet) + + +static void ZstdDec1_Construct(CZstdDec1 *p) +{ + #ifdef Z7_ZSTD_DEC_PRINT_TABLE + Print_Predef(6, NUM_LL_SYMBOLS, SEQ_LL_PREDEF_DIST, k_PredefRecords_LL); + Print_Predef(5, NUM_OFFSET_SYMBOLS_PREDEF, SEQ_OFFSET_PREDEF_DIST, k_PredefRecords_OF); + Print_Predef(6, NUM_ML_SYMBOLS, SEQ_ML_PREDEF_DIST, k_PredefRecords_ML); + #endif + + p->win = NULL; + p->cycSize = 0; + p->literalsBase = NULL; + #ifdef Z7_ZSTD_DEC_USE_BASES_IN_OBJECT + FILL_LOC_BASES_ALL + #endif +} + + +static void ZstdDec1_Init(CZstdDec1 *p) +{ + p->reps[0] = 1; + p->reps[1] = 4; + p->reps[2] = 8; + // p->seqTables_wereSet = False; + p->ml_accuracy = SEQ_TABLES_WERE_NOT_SET_ml_accuracy; + p->litHuf_wasSet = False; + p->totalOutCheck = 0; +} + + + +#ifdef MY_CPU_LE_UNALIGN + #define Z7_ZSTD_DEC_USE_UNALIGNED_COPY +#endif + +#ifdef Z7_ZSTD_DEC_USE_UNALIGNED_COPY + + #define COPY_CHUNK_SIZE 16 + + #define COPY_CHUNK_4_2(dest, src) \ + { \ + ((UInt32 *)(void *)dest)[0] = ((const UInt32 *)(const void *)src)[0]; \ + ((UInt32 *)(void *)dest)[1] = ((const UInt32 *)(const void *)src)[1]; \ + src += 4 * 2; \ + dest += 4 * 2; \ + } + + /* sse2 doesn't help here in GCC and CLANG. + so we disabled sse2 here */ + /* + #if defined(MY_CPU_AMD64) + #define Z7_ZSTD_DEC_USE_SSE2 + #elif defined(MY_CPU_X86) + #if defined(_MSC_VER) && _MSC_VER >= 1300 && defined(_M_IX86_FP) && (_M_IX86_FP >= 2) \ + || defined(__SSE2__) \ + // || 1 == 1 // for debug only + #define Z7_ZSTD_DEC_USE_SSE2 + #endif + #endif + */ + + #if defined(MY_CPU_ARM64) + #define COPY_OFFSET_MIN 16 + #define COPY_CHUNK1(dest, src) \ + { \ + vst1q_u8((uint8_t *)(void *)dest, \ + vld1q_u8((const uint8_t *)(const void *)src)); \ + src += 16; \ + dest += 16; \ + } + + #define COPY_CHUNK(dest, src) \ + { \ + COPY_CHUNK1(dest, src) \ + if ((len -= COPY_CHUNK_SIZE) == 0) break; \ + COPY_CHUNK1(dest, src) \ + } + + #elif defined(Z7_ZSTD_DEC_USE_SSE2) + #include // sse2 + #define COPY_OFFSET_MIN 16 + + #define COPY_CHUNK1(dest, src) \ + { \ + _mm_storeu_si128((__m128i *)(void *)dest, \ + _mm_loadu_si128((const __m128i *)(const void *)src)); \ + src += 16; \ + dest += 16; \ + } + + #define COPY_CHUNK(dest, src) \ + { \ + COPY_CHUNK1(dest, src) \ + if ((len -= COPY_CHUNK_SIZE) == 0) break; \ + COPY_CHUNK1(dest, src) \ + } + + #elif defined(MY_CPU_64BIT) + #define COPY_OFFSET_MIN 8 + + #define COPY_CHUNK(dest, src) \ + { \ + ((UInt64 *)(void *)dest)[0] = ((const UInt64 *)(const void *)src)[0]; \ + ((UInt64 *)(void *)dest)[1] = ((const UInt64 *)(const void *)src)[1]; \ + src += 8 * 2; \ + dest += 8 * 2; \ + } + + #else + #define COPY_OFFSET_MIN 4 + + #define COPY_CHUNK(dest, src) \ + { \ + COPY_CHUNK_4_2(dest, src); \ + COPY_CHUNK_4_2(dest, src); \ + } + + #endif +#endif + + +#ifndef COPY_CHUNK_SIZE + #define COPY_OFFSET_MIN 4 + #define COPY_CHUNK_SIZE 8 + #define COPY_CHUNK_2(dest, src) \ + { \ + const Byte a0 = src[0]; \ + const Byte a1 = src[1]; \ + dest[0] = a0; \ + dest[1] = a1; \ + src += 2; \ + dest += 2; \ + } + #define COPY_CHUNK(dest, src) \ + { \ + COPY_CHUNK_2(dest, src) \ + COPY_CHUNK_2(dest, src) \ + COPY_CHUNK_2(dest, src) \ + COPY_CHUNK_2(dest, src) \ + } +#endif + + +#define COPY_PREPARE \ + len += (COPY_CHUNK_SIZE - 1); \ + len &= ~(size_t)(COPY_CHUNK_SIZE - 1); \ + { if (len > rem) \ + { len = rem; \ + rem &= (COPY_CHUNK_SIZE - 1); \ + if (rem) { \ + len -= rem; \ + Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE \ + do *dest++ = *src++; while (--rem); \ + if (len == 0) return; }}} + +#define COPY_CHUNKS \ +{ \ + Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE \ + do { COPY_CHUNK(dest, src) } \ + while (len -= COPY_CHUNK_SIZE); \ +} + +// (len != 0) +// (len <= rem) +static +Z7_FORCE_INLINE +// Z7_ATTRIB_NO_VECTOR +void CopyLiterals(Byte *dest, Byte const *src, size_t len, size_t rem) +{ + COPY_PREPARE + COPY_CHUNKS +} + + +/* we can define Z7_STD_DEC_USE_AFTER_CYC_BUF, if we want to use additional + space after cycSize that can be used to reduce the code in CopyMatch(): */ +// for debug: +// #define Z7_STD_DEC_USE_AFTER_CYC_BUF + +/* +CopyMatch() +if wrap (offset > winPos) +{ + then we have at least (COPY_CHUNK_SIZE) avail in (dest) before we will overwrite (src): + (cycSize >= offset + COPY_CHUNK_SIZE) + if defined(Z7_STD_DEC_USE_AFTER_CYC_BUF) + we are allowed to read win[cycSize + COPY_CHUNK_SIZE - 1], +} +(len != 0) +*/ +static +Z7_FORCE_INLINE +// Z7_ATTRIB_NO_VECTOR +void CopyMatch(size_t offset, size_t len, + Byte *win, size_t winPos, size_t rem, const size_t cycSize) +{ + Byte *dest = win + winPos; + const Byte *src; + // STAT_INC(g_NumCopy) + + if (offset > winPos) + { + size_t back = offset - winPos; + // src = win + cycSize - back; + // cycSize -= offset; + STAT_INC(g_NumOver) + src = dest + (cycSize - offset); + // (src >= dest) here + #ifdef Z7_STD_DEC_USE_AFTER_CYC_BUF + if (back < len) + { + #else + if (back < len + (COPY_CHUNK_SIZE - 1)) + { + if (back >= len) + { + Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE + do + *dest++ = *src++; + while (--len); + return; + } + #endif + // back < len + STAT_INC(g_NumOver2) + len -= back; + rem -= back; + Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE + do + *dest++ = *src++; + while (--back); + src = dest - offset; + // src = win; + // we go to MAIN-COPY + } + } + else + src = dest - offset; + + // len != 0 + // do *dest++ = *src++; while (--len); return; + + // --- MAIN COPY --- + // if (src >= dest), then ((size_t)(src - dest) >= COPY_CHUNK_SIZE) + // so we have at least COPY_CHUNK_SIZE space before overlap for writing. + COPY_PREPARE + + /* now (len == COPY_CHUNK_SIZE * x) + so we can unroll for aligned copy */ + { + // const unsigned b0 = src[0]; + // (COPY_OFFSET_MIN >= 4) + + if (offset >= COPY_OFFSET_MIN) + { + COPY_CHUNKS + // return; + } + else + #if (COPY_OFFSET_MIN > 4) + #if COPY_CHUNK_SIZE < 8 + #error Stop_Compiling_Bad_COPY_CHUNK_SIZE + #endif + if (offset >= 4) + { + Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE + do + { + COPY_CHUNK_4_2(dest, src) + #if COPY_CHUNK_SIZE != 16 + if (len == 8) break; + #endif + COPY_CHUNK_4_2(dest, src) + } + while (len -= 16); + // return; + } + else + #endif + { + // (offset < 4) + const unsigned b0 = src[0]; + if (offset < 2) + { + #if defined(Z7_ZSTD_DEC_USE_UNALIGNED_COPY) && (COPY_CHUNK_SIZE == 16) + #if defined(MY_CPU_64BIT) + { + const UInt64 v64 = (UInt64)b0 * 0x0101010101010101; + Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE + do + { + ((UInt64 *)(void *)dest)[0] = v64; + ((UInt64 *)(void *)dest)[1] = v64; + dest += 16; + } + while (len -= 16); + } + #else + { + UInt32 v = b0; + v |= v << 8; + v |= v << 16; + do + { + ((UInt32 *)(void *)dest)[0] = v; + ((UInt32 *)(void *)dest)[1] = v; + dest += 8; + ((UInt32 *)(void *)dest)[0] = v; + ((UInt32 *)(void *)dest)[1] = v; + dest += 8; + } + while (len -= 16); + } + #endif + #else + do + { + dest[0] = (Byte)b0; + dest[1] = (Byte)b0; + dest += 2; + dest[0] = (Byte)b0; + dest[1] = (Byte)b0; + dest += 2; + } + while (len -= 4); + #endif + } + else if (offset == 2) + { + const Byte b1 = src[1]; + { + do + { + dest[0] = (Byte)b0; + dest[1] = b1; + dest += 2; + } + while (len -= 2); + } + } + else // (offset == 3) + { + const Byte *lim = dest + len - 2; + const Byte b1 = src[1]; + const Byte b2 = src[2]; + do + { + dest[0] = (Byte)b0; + dest[1] = b1; + dest[2] = b2; + dest += 3; + } + while (dest < lim); + lim++; // points to last byte that must be written + if (dest <= lim) + { + *dest = (Byte)b0; + if (dest != lim) + dest[1] = b1; + } + } + } + } +} + + + +#define UPDATE_TOTAL_OUT(p, size) \ +{ \ + size_t _toc = (p)->totalOutCheck + (size); \ + const size_t _ws = (p)->winSize; \ + if (_toc >= _ws) _toc = _ws; \ + (p)->totalOutCheck = _toc; \ +} + + +#if defined(MY_CPU_64BIT) && defined(MY_CPU_LE_UNALIGN) +// we can disable it for debug: +#define Z7_ZSTD_DEC_USE_64BIT_LOADS +#endif +// #define Z7_ZSTD_DEC_USE_64BIT_LOADS // for debug : slow in 32-bit + +// SEQ_SRC_OFFSET: how many bytes (src) (seqSrc) was moved back from original value. +// we need (SEQ_SRC_OFFSET != 0) for optimized memory access +#ifdef Z7_ZSTD_DEC_USE_64BIT_LOADS + #define SEQ_SRC_OFFSET 7 +#else + #define SEQ_SRC_OFFSET 3 +#endif +#define SRC_PLUS_FOR_4BYTES(bitOffset) (SEQ_SRC_OFFSET - 3) + ((CBitCtr_signed)(bitOffset) >> 3) +#define BIT_OFFSET_7BITS(bitOffset) ((unsigned)(bitOffset) & 7) +/* + if (BIT_OFFSET_DELTA_BITS == 0) : bitOffset == number_of_unprocessed_bits + if (BIT_OFFSET_DELTA_BITS == 1) : bitOffset == number_of_unprocessed_bits - 1 + and we can read 1 bit more in that mode : (8 * n + 1). +*/ +// #define BIT_OFFSET_DELTA_BITS 0 +#define BIT_OFFSET_DELTA_BITS 1 +#if BIT_OFFSET_DELTA_BITS == 1 + #define GET_SHIFT_FROM_BOFFS7(boff7) (7 ^ (boff7)) +#else + #define GET_SHIFT_FROM_BOFFS7(boff7) (8 - BIT_OFFSET_DELTA_BITS - (boff7)) +#endif + +#define UPDATE_BIT_OFFSET(bitOffset, numBits) \ + (bitOffset) -= (CBitCtr)(numBits); + +#define GET_SHIFT(bitOffset) GET_SHIFT_FROM_BOFFS7(BIT_OFFSET_7BITS(bitOffset)) + + +#if defined(Z7_ZSTD_DEC_USE_64BIT_LOADS) + #if (NUM_OFFSET_SYMBOLS_MAX - BIT_OFFSET_DELTA_BITS < 32) + /* if (NUM_OFFSET_SYMBOLS_MAX == 32 && BIT_OFFSET_DELTA_BITS == 1), + we have depth 31 + 9 + 9 + 8 = 57 bits that can b read with single read. */ + #define Z7_ZSTD_DEC_USE_64BIT_PRELOAD_OF + #endif + #ifndef Z7_ZSTD_DEC_USE_64BIT_PRELOAD_OF + #if (BIT_OFFSET_DELTA_BITS == 1) + /* if (winLimit - winPos <= (kBlockSizeMax = (1 << 17))) + { + the case (16 bits literal extra + 16 match extra) is not possible + in correct stream. So error will be detected for (16 + 16) case. + And longest correct sequence after offset reading is (31 + 9 + 9 + 8 = 57 bits). + So we can use just one 64-bit load here in that case. + } + */ + #define Z7_ZSTD_DEC_USE_64BIT_PRELOAD_ML + #endif + #endif +#endif + + +#if !defined(Z7_ZSTD_DEC_USE_64BIT_LOADS) || \ + (!defined(Z7_ZSTD_DEC_USE_64BIT_PRELOAD_OF) && \ + !defined(Z7_ZSTD_DEC_USE_64BIT_PRELOAD_ML)) +// in : (0 < bits <= (24 or 25)): +#define STREAM_READ_BITS(dest, bits) \ +{ \ + GET32(dest, src + SRC_PLUS_FOR_4BYTES(bitOffset)) \ + dest <<= GET_SHIFT(bitOffset); \ + UPDATE_BIT_OFFSET(bitOffset, bits) \ + dest >>= 32 - bits; \ +} +#endif + + +#define FSE_Peek_1(table, state) table[state] + +#define STATE_VAR(name) state_ ## name + +// in : (0 <= accuracy <= (24 or 25)) +#define FSE_INIT_STATE(name, cond) \ +{ \ + UInt32 r; \ + const unsigned bits = p->name ## _accuracy; \ + GET32(r, src + SRC_PLUS_FOR_4BYTES(bitOffset)) \ + r <<= GET_SHIFT(bitOffset); \ + r >>= 1; \ + r >>= 31 ^ bits; \ + UPDATE_BIT_OFFSET(bitOffset, bits) \ + cond \ + STATE_VAR(name) = FSE_Peek_1(FSE_TABLE(name), r); \ + /* STATE_VAR(name) = dest << 16; */ \ +} + + +#define FSE_Peek_Plus(name, r) \ + STATE_VAR(name) = FSE_Peek_1(FSE_TABLE(name), \ + GET_FSE_REC_STATE(STATE_VAR(name)) + r); + +#define LZ_LOOP_ERROR_EXIT { return SZ_ERROR_DATA; } + +#define BO_OVERFLOW_CHECK \ + { if ((CBitCtr_signed)bitOffset < 0) LZ_LOOP_ERROR_EXIT } + + +#ifdef Z7_ZSTD_DEC_USE_64BIT_LOADS + +#define GET64(dest, p) { const Byte *ptr = p; dest = GetUi64(ptr); } + +#define FSE_PRELOAD \ +{ \ + GET64(v, src - 4 + SRC_PLUS_FOR_4BYTES(bitOffset)) \ + v <<= GET_SHIFT(bitOffset); \ +} + +#define FSE_UPDATE_STATE_2(name, cond) \ +{ \ + const unsigned bits = GET_FSE_REC_LEN(STATE_VAR(name)); \ + UInt64 r = v; \ + v <<= bits; \ + r >>= 1; \ + UPDATE_BIT_OFFSET(bitOffset, bits) \ + cond \ + r >>= 63 ^ bits; \ + FSE_Peek_Plus(name, r); \ +} + +#define FSE_UPDATE_STATES \ + FSE_UPDATE_STATE_2 (ll, {} ) \ + FSE_UPDATE_STATE_2 (ml, {} ) \ + FSE_UPDATE_STATE_2 (of, BO_OVERFLOW_CHECK) \ + +#else // Z7_ZSTD_DEC_USE_64BIT_LOADS + +// it supports 8 bits accuracy for any code +// it supports 9 bits accuracy, if (BIT_OFFSET_DELTA_BITS == 1) +#define FSE_UPDATE_STATE_0(name, cond) \ +{ \ + UInt32 r; \ + const unsigned bits = GET_FSE_REC_LEN(STATE_VAR(name)); \ + GET16(r, src + 2 + SRC_PLUS_FOR_4BYTES(bitOffset)) \ + r >>= (bitOffset & 7); \ + r &= (1 << (8 + BIT_OFFSET_DELTA_BITS)) - 1; \ + UPDATE_BIT_OFFSET(bitOffset, bits) \ + cond \ + r >>= (8 + BIT_OFFSET_DELTA_BITS) - bits; \ + FSE_Peek_Plus(name, r); \ +} + +// for debug (slow): +// #define Z7_ZSTD_DEC_USE_FSE_FUSION_FORCE +#if BIT_OFFSET_DELTA_BITS == 0 || defined(Z7_ZSTD_DEC_USE_FSE_FUSION_FORCE) + #define Z7_ZSTD_DEC_USE_FSE_FUSION +#endif + +#ifdef Z7_ZSTD_DEC_USE_FSE_FUSION +#define FSE_UPDATE_STATE_1(name) \ +{ UInt32 rest2; \ +{ \ + UInt32 r; \ + unsigned bits; \ + GET32(r, src + SRC_PLUS_FOR_4BYTES(bitOffset)) \ + bits = GET_FSE_REC_LEN(STATE_VAR(name)); \ + r <<= GET_SHIFT(bitOffset); \ + rest2 = r << bits; \ + r >>= 1; \ + UPDATE_BIT_OFFSET(bitOffset, bits) \ + r >>= 31 ^ bits; \ + FSE_Peek_Plus(name, r); \ +} + +#define FSE_UPDATE_STATE_3(name) \ +{ \ + const unsigned bits = GET_FSE_REC_LEN(STATE_VAR(name)); \ + rest2 >>= 1; \ + UPDATE_BIT_OFFSET(bitOffset, bits) \ + rest2 >>= 31 ^ bits; \ + FSE_Peek_Plus(name, rest2); \ +}} + +#define FSE_UPDATE_STATES \ + FSE_UPDATE_STATE_1 (ll) \ + FSE_UPDATE_STATE_3 (ml) \ + FSE_UPDATE_STATE_0 (of, BO_OVERFLOW_CHECK) \ + +#else // Z7_ZSTD_DEC_USE_64BIT_LOADS + +#define FSE_UPDATE_STATES \ + FSE_UPDATE_STATE_0 (ll, {} ) \ + FSE_UPDATE_STATE_0 (ml, {} ) \ + FSE_UPDATE_STATE_0 (of, BO_OVERFLOW_CHECK) \ + +#endif // Z7_ZSTD_DEC_USE_FSE_FUSION +#endif // Z7_ZSTD_DEC_USE_64BIT_LOADS + + + +typedef struct +{ + UInt32 numSeqs; + UInt32 literalsLen; + const Byte *literals; +} +CZstdDec1_Vars; + + +// if (BIT_OFFSET_DELTA_BITS != 0), we need (BIT_OFFSET_DELTA_BYTES > 0) +#define BIT_OFFSET_DELTA_BYTES BIT_OFFSET_DELTA_BITS + +/* if (NUM_OFFSET_SYMBOLS_MAX == 32) + max_seq_bit_length = (31) + 16 + 16 + 9 + 8 + 9 = 89 bits + if defined(Z7_ZSTD_DEC_USE_64BIT_PRELOAD_OF) we have longest backward + lookahead offset, and we read UInt64 after literal_len reading. + if (BIT_OFFSET_DELTA_BITS == 1 && NUM_OFFSET_SYMBOLS_MAX == 32) + MAX_BACKWARD_DEPTH = 16 bytes +*/ +#define MAX_BACKWARD_DEPTH \ + ((NUM_OFFSET_SYMBOLS_MAX - 1 + 16 + 16 + 7) / 8 + 7 + BIT_OFFSET_DELTA_BYTES) + +/* srcLen != 0 + src == real_data_ptr - SEQ_SRC_OFFSET - BIT_OFFSET_DELTA_BYTES + if defined(Z7_ZSTD_DEC_USE_64BIT_PRELOAD_ML) then + (winLimit - p->winPos <= (1 << 17)) is required +*/ +static +Z7_NO_INLINE +// Z7_ATTRIB_NO_VECTOR +SRes Decompress_Sequences(CZstdDec1 * const p, + const Byte *src, const size_t srcLen, + const size_t winLimit, + const CZstdDec1_Vars * const vars) +{ +#ifdef Z7_ZSTD_DEC_USE_BASES_LOCAL + SEQ_EXTRA_TABLES(a_) +#endif + + // for debug: + // #define Z7_ZSTD_DEC_USE_LOCAL_FSE_TABLES +#ifdef Z7_ZSTD_DEC_USE_LOCAL_FSE_TABLES + #define FSE_TABLE(n) fse. n + const CZstdDecFseTables fse = p->fse; + /* + CZstdDecFseTables fse; + #define COPY_FSE_TABLE(n) \ + memcpy(fse. n, p->fse. n, (size_t)4 << p-> n ## _accuracy); + COPY_FSE_TABLE(of) + COPY_FSE_TABLE(ll) + COPY_FSE_TABLE(ml) + */ +#else + #define FSE_TABLE(n) (p->fse. n) +#endif + +#ifdef Z7_ZSTD_DEC_USE_BASES_LOCAL + FILL_LOC_BASES_ALL +#endif + + { + unsigned numSeqs = vars->numSeqs; + const Byte *literals = vars->literals; + ptrdiff_t literalsLen = (ptrdiff_t)vars->literalsLen; + Byte * const win = p->win; + size_t winPos = p->winPos; + const size_t cycSize = p->cycSize; + size_t totalOutCheck = p->totalOutCheck; + const size_t winSize = p->winSize; + size_t reps_0 = p->reps[0]; + size_t reps_1 = p->reps[1]; + size_t reps_2 = p->reps[2]; + UInt32 STATE_VAR(ll), STATE_VAR(of), STATE_VAR(ml); + CBitCtr bitOffset; + + SET_bitOffset_TO_PAD (bitOffset, src + SEQ_SRC_OFFSET, srcLen + BIT_OFFSET_DELTA_BYTES) + + bitOffset -= BIT_OFFSET_DELTA_BITS; + + FSE_INIT_STATE(ll, {} ) + FSE_INIT_STATE(of, {} ) + FSE_INIT_STATE(ml, BO_OVERFLOW_CHECK) + + for (;;) + { + size_t matchLen; + #ifdef Z7_ZSTD_DEC_USE_64BIT_LOADS + UInt64 v; + #endif + + #ifdef Z7_ZSTD_DEC_USE_64BIT_PRELOAD_OF + FSE_PRELOAD + #endif + + // if (of_code == 0) + if ((Byte)STATE_VAR(of) == 0) + { + if (GET_FSE_REC_SYM(STATE_VAR(ll)) == 0) + { + const size_t offset = reps_1; + reps_1 = reps_0; + reps_0 = offset; + STAT_INC(g_Num_Rep1) + } + STAT_UPDATE(else g_Num_Rep0++;) + } + else + { + const unsigned of_code = (Byte)STATE_VAR(of); + + #ifdef Z7_ZSTD_DEC_USE_64BIT_LOADS + #if !defined(Z7_ZSTD_DEC_USE_64BIT_PRELOAD_OF) + FSE_PRELOAD + #endif + #else + UInt32 v; + { + const Byte *src4 = src + SRC_PLUS_FOR_4BYTES(bitOffset); + const unsigned skip = GET_SHIFT(bitOffset); + GET32(v, src4) + v <<= skip; + v |= (UInt32)src4[-1] >> (8 - skip); + } + #endif + + UPDATE_BIT_OFFSET(bitOffset, of_code) + + if (of_code == 1) + { + // read 1 bit + #if defined(Z7_MSC_VER_ORIGINAL) || defined(MY_CPU_X86_OR_AMD64) + #ifdef Z7_ZSTD_DEC_USE_64BIT_LOADS + #define CHECK_HIGH_BIT_64(a) ((Int64)(UInt64)(a) < 0) + #else + #define CHECK_HIGH_BIT_32(a) ((Int32)(UInt32)(a) < 0) + #endif + #else + #ifdef Z7_ZSTD_DEC_USE_64BIT_LOADS + #define CHECK_HIGH_BIT_64(a) ((UInt64)(a) & ((UInt64)1 << 63)) + #else + #define CHECK_HIGH_BIT_32(a) ((UInt32)(a) & ((UInt32)1 << 31)) + #endif + #endif + + if + #ifdef Z7_ZSTD_DEC_USE_64BIT_LOADS + CHECK_HIGH_BIT_64 (((UInt64)GET_FSE_REC_SYM(STATE_VAR(ll)) - 1) ^ v) + #else + CHECK_HIGH_BIT_32 (((UInt32)GET_FSE_REC_SYM(STATE_VAR(ll)) - 1) ^ v) + #endif + { + v <<= 1; + { + const size_t offset = reps_2; + reps_2 = reps_1; + reps_1 = reps_0; + reps_0 = offset; + STAT_INC(g_Num_Rep2) + } + } + else + { + if (GET_FSE_REC_SYM(STATE_VAR(ll)) == 0) + { + // litLen == 0 && bit == 1 + STAT_INC(g_Num_Rep3) + v <<= 1; + reps_2 = reps_1; + reps_1 = reps_0; + if (--reps_0 == 0) + { + // LZ_LOOP_ERROR_EXIT + // original-zstd decoder : input is corrupted; force offset to 1 + // reps_0 = 1; + reps_0++; + } + } + else + { + // litLen != 0 && bit == 0 + v <<= 1; + { + const size_t offset = reps_1; + reps_1 = reps_0; + reps_0 = offset; + STAT_INC(g_Num_Rep1) + } + } + } + } + else + { + // (2 <= of_code) + // if (of_code >= 32) LZ_LOOP_ERROR_EXIT // optional check + // we don't allow (of_code >= 32) cases in another code + reps_2 = reps_1; + reps_1 = reps_0; + reps_0 = ((size_t)1 << of_code) - 3 + (size_t) + #ifdef Z7_ZSTD_DEC_USE_64BIT_LOADS + (v >> (64 - of_code)); + v <<= of_code; + #else + (v >> (32 - of_code)); + #endif + } + } + + #ifdef Z7_ZSTD_DEC_USE_64BIT_PRELOAD_ML + FSE_PRELOAD + #endif + + matchLen = (size_t)GET_FSE_REC_SYM(STATE_VAR(ml)) + #ifndef Z7_ZSTD_DEC_USE_ML_PLUS3 + + MATCH_LEN_MIN + #endif + ; + { + { + if (matchLen >= 32 + MATCH_LEN_MIN) // if (state_ml & 0x20) + { + const unsigned extra = BASES_TABLE(SEQ_ML_EXTRA) [(size_t)matchLen - MATCH_LEN_MIN]; + matchLen = BASES_TABLE(SEQ_ML_BASES) [(size_t)matchLen - MATCH_LEN_MIN]; + #if defined(Z7_ZSTD_DEC_USE_64BIT_LOADS) && \ + (defined(Z7_ZSTD_DEC_USE_64BIT_PRELOAD_ML) || \ + defined(Z7_ZSTD_DEC_USE_64BIT_PRELOAD_OF)) + { + UPDATE_BIT_OFFSET(bitOffset, extra) + matchLen += (size_t)(v >> (64 - extra)); + #if defined(Z7_ZSTD_DEC_USE_64BIT_PRELOAD_OF) + FSE_PRELOAD + #else + v <<= extra; + #endif + } + #else + { + UInt32 v32; + STREAM_READ_BITS(v32, extra) + matchLen += v32; + } + #endif + STAT_INC(g_Num_Match) + } + } + } + + #if defined(Z7_ZSTD_DEC_USE_64BIT_LOADS) && \ + !defined(Z7_ZSTD_DEC_USE_64BIT_PRELOAD_OF) && \ + !defined(Z7_ZSTD_DEC_USE_64BIT_PRELOAD_ML) + FSE_PRELOAD + #endif + + { + size_t litLen = GET_FSE_REC_SYM(STATE_VAR(ll)); + if (litLen) + { + // if (STATE_VAR(ll) & 0x70) + if (litLen >= 16) + { + const unsigned extra = BASES_TABLE(SEQ_LL_EXTRA) [litLen]; + litLen = BASES_TABLE(SEQ_LL_BASES) [litLen]; + #ifdef Z7_ZSTD_DEC_USE_64BIT_LOADS + { + UPDATE_BIT_OFFSET(bitOffset, extra) + litLen += (size_t)(v >> (64 - extra)); + #if defined(Z7_ZSTD_DEC_USE_64BIT_PRELOAD_OF) + FSE_PRELOAD + #else + v <<= extra; + #endif + } + #else + { + UInt32 v32; + STREAM_READ_BITS(v32, extra) + litLen += v32; + } + #endif + STAT_INC(g_Num_LitsBig) + } + + if ((literalsLen -= (ptrdiff_t)litLen) < 0) + LZ_LOOP_ERROR_EXIT + totalOutCheck += litLen; + { + const size_t rem = winLimit - winPos; + if (litLen > rem) + LZ_LOOP_ERROR_EXIT + { + const Byte *literals_temp = literals; + Byte *d = win + winPos; + literals += litLen; + winPos += litLen; + CopyLiterals(d, literals_temp, litLen, rem); + } + } + } + STAT_UPDATE(else g_Num_Lit0++;) + } + + #define COPY_MATCH \ + { if (reps_0 > winSize || reps_0 > totalOutCheck) LZ_LOOP_ERROR_EXIT \ + totalOutCheck += matchLen; \ + { const size_t rem = winLimit - winPos; \ + if (matchLen > rem) LZ_LOOP_ERROR_EXIT \ + { const size_t winPos_temp = winPos; \ + winPos += matchLen; \ + CopyMatch(reps_0, matchLen, win, winPos_temp, rem, cycSize); }}} + + if (--numSeqs == 0) + { + COPY_MATCH + break; + } + FSE_UPDATE_STATES + COPY_MATCH + } // for + + if ((CBitCtr_signed)bitOffset != BIT_OFFSET_DELTA_BYTES * 8 - BIT_OFFSET_DELTA_BITS) + return SZ_ERROR_DATA; + + if (literalsLen) + { + const size_t rem = winLimit - winPos; + if ((size_t)literalsLen > rem) + return SZ_ERROR_DATA; + { + Byte *d = win + winPos; + winPos += (size_t)literalsLen; + totalOutCheck += (size_t)literalsLen; + CopyLiterals + // memcpy + (d, literals, (size_t)literalsLen, rem); + } + } + if (totalOutCheck >= winSize) + totalOutCheck = winSize; + p->totalOutCheck = totalOutCheck; + p->winPos = winPos; + p->reps[0] = (CZstdDecOffset)reps_0; + p->reps[1] = (CZstdDecOffset)reps_1; + p->reps[2] = (CZstdDecOffset)reps_2; + } + return SZ_OK; +} + + +// for debug: define to check that ZstdDec1_NeedTempBufferForInput() works correctly: +// #define Z7_ZSTD_DEC_USE_CHECK_OF_NEED_TEMP // define it for debug only +#ifdef Z7_ZSTD_DEC_USE_CHECK_OF_NEED_TEMP +static unsigned g_numSeqs; +#endif + + +#define k_LitBlockType_Flag_RLE_or_Treeless 1 +#define k_LitBlockType_Flag_Compressed 2 + +// outLimit : is strong limit +// outLimit <= ZstdDec1_GET_BLOCK_SIZE_LIMIT(p) +// inSize != 0 +static +Z7_NO_INLINE +SRes ZstdDec1_DecodeBlock(CZstdDec1 *p, + const Byte *src, SizeT inSize, SizeT afterAvail, + const size_t outLimit) +{ + CZstdDec1_Vars vars; + vars.literals = p->literalsBase; + { + const unsigned b0 = *src++; + UInt32 numLits, compressedSize; + const Byte *litStream; + Byte *literalsDest; + inSize--; + + if ((b0 & k_LitBlockType_Flag_Compressed) == 0) + { + // we need at least one additional byte for (numSeqs). + // so we check for that additional byte in conditions. + numLits = b0 >> 3; + if (b0 & 4) + { + UInt32 v; + if (inSize < 1 + 1) // we need at least 1 byte here and 1 byte for (numSeqs). + return SZ_ERROR_DATA; + numLits >>= 1; + v = GetUi16(src); + src += 2; + inSize -= 2; + if ((b0 & 8) == 0) + { + src--; + inSize++; + v = (Byte)v; + } + numLits += v << 4; + } + compressedSize = 1; + if ((b0 & k_LitBlockType_Flag_RLE_or_Treeless) == 0) + compressedSize = numLits; + } + else if (inSize < 4) + return SZ_ERROR_DATA; + else + { + const unsigned mode4Streams = b0 & 0xc; + const unsigned numBytes = (3 * mode4Streams + 32) >> 4; + const unsigned numBits = 4 * numBytes - 2; + const UInt32 mask = ((UInt32)16 << numBits) - 1; + compressedSize = GetUi32(src); + numLits = (( + #ifdef MY_CPU_LE_UNALIGN + GetUi32(src - 1) + #else + ((compressedSize << 8) + b0) + #endif + ) >> 4) & mask; + src += numBytes; + inSize -= numBytes; + compressedSize >>= numBits; + compressedSize &= mask; + /* + if (numLits != 0) printf("inSize = %7u num_lits=%7u compressed=%7u ratio = %u ratio2 = %u\n", + i1, numLits, (unsigned)compressedSize * 1, (unsigned)compressedSize * 100 / numLits, + (unsigned)numLits * 100 / (unsigned)inSize); + } + */ + if (compressedSize == 0) + return SZ_ERROR_DATA; // (compressedSize == 0) is not allowed + } + + STAT_UPDATE(g_Num_Lits += numLits;) + + vars.literalsLen = numLits; + + if (compressedSize >= inSize) + return SZ_ERROR_DATA; + litStream = src; + src += compressedSize; + inSize -= compressedSize; + // inSize != 0 + { + UInt32 numSeqs = *src++; + inSize--; + if (numSeqs > 127) + { + UInt32 b1; + if (inSize == 0) + return SZ_ERROR_DATA; + numSeqs -= 128; + b1 = *src++; + inSize--; + if (numSeqs == 127) + { + if (inSize == 0) + return SZ_ERROR_DATA; + numSeqs = (UInt32)(*src++) + 127; + inSize--; + } + numSeqs = (numSeqs << 8) + b1; + } + if (numSeqs * MATCH_LEN_MIN + numLits > outLimit) + return SZ_ERROR_DATA; + vars.numSeqs = numSeqs; + + STAT_UPDATE(g_NumSeqs_total += numSeqs;) + /* + #ifdef SHOW_STAT + printf("\n %5u : %8u, %8u : %5u", (int)g_Num_Blocks_Compressed, (int)numSeqs, (int)g_NumSeqs_total, + (int)g_NumSeqs_total / g_Num_Blocks_Compressed); + #endif + // printf("\nnumSeqs2 = %d", numSeqs); + */ + #ifdef Z7_ZSTD_DEC_USE_CHECK_OF_NEED_TEMP + if (numSeqs != g_numSeqs) return SZ_ERROR_DATA; // for debug + #endif + if (numSeqs == 0) + { + if (inSize != 0) + return SZ_ERROR_DATA; + literalsDest = p->win + p->winPos; + } + else + literalsDest = p->literalsBase; + } + + if ((b0 & k_LitBlockType_Flag_Compressed) == 0) + { + if (b0 & k_LitBlockType_Flag_RLE_or_Treeless) + { + memset(literalsDest, litStream[0], numLits); + if (vars.numSeqs) + { + // literalsDest == p->literalsBase == vars.literals + #if COPY_CHUNK_SIZE > 1 + memset(p->literalsBase + numLits, 0, COPY_CHUNK_SIZE); + #endif + } + } + else + { + // unsigned y; + // for (y = 0; y < 10000; y++) + memcpy(literalsDest, litStream, numLits); + if (vars.numSeqs) + { + /* we need up to (15 == COPY_CHUNK_SIZE - 1) space for optimized CopyLiterals(). + If we have additional space in input stream after literals stream, + we use direct copy of rar literals in input stream */ + if ((size_t)(src + inSize - litStream) - numLits + afterAvail >= (COPY_CHUNK_SIZE - 1)) + vars.literals = litStream; + else + { + // literalsDest == p->literalsBase == vars.literals + #if COPY_CHUNK_SIZE > 1 + /* CopyLiterals(): + 1) we don't want reading non-initialized data + 2) we will copy only zero byte after literals buffer */ + memset(p->literalsBase + numLits, 0, COPY_CHUNK_SIZE); + #endif + } + } + } + } + else + { + CInBufPair hufStream; + hufStream.ptr = litStream; + hufStream.len = compressedSize; + + if ((b0 & k_LitBlockType_Flag_RLE_or_Treeless) == 0) + { + // unsigned y = 100; CInBufPair hs2 = hufStream; do { hufStream = hs2; + RINOK(Huf_DecodeTable(&p->huf, &hufStream)) + p->litHuf_wasSet = True; + // } while (--y); + } + else if (!p->litHuf_wasSet) + return SZ_ERROR_DATA; + + { + // int yyy; for (yyy = 0; yyy < 34; yyy++) { + SRes sres; + if ((b0 & 0xc) == 0) // mode4Streams + sres = Huf_Decompress_1stream((const Byte *)(const void *)p->huf.table64, + hufStream.ptr - HUF_SRC_OFFSET, hufStream.len, literalsDest, numLits); + else + { + // 6 bytes for the jump table + 4x1 bytes of end-padding Bytes) + if (hufStream.len < 6 + 4) + return SZ_ERROR_DATA; + // the condition from original-zstd decoder: + #define Z7_ZSTD_MIN_LITERALS_FOR_4_STREAMS 6 + if (numLits < Z7_ZSTD_MIN_LITERALS_FOR_4_STREAMS) + return SZ_ERROR_DATA; + sres = Huf_Decompress_4stream((const Byte *)(const void *)p->huf.table64, + hufStream.ptr + (6 - HUF_SRC_OFFSET), hufStream.len, literalsDest, numLits); + } + RINOK(sres) + // } + } + } + + if (vars.numSeqs == 0) + { + p->winPos += numLits; + return SZ_OK; + } + } + { + CInBufPair in; + unsigned mode; + unsigned seqMode; + + in.ptr = src; + in.len = inSize; + if (in.len == 0) + return SZ_ERROR_DATA; + in.len--; + mode = *in.ptr++; + if (mode & 3) // Reserved bits + return SZ_ERROR_DATA; + + seqMode = (mode >> 6); + if (seqMode == k_SeqMode_Repeat) + { if (!IS_SEQ_TABLES_WERE_SET(p)) return SZ_ERROR_DATA; } + else RINOK(FSE_Decode_SeqTable( + p->fse.ll, + &in, + 6, // predefAccuracy + &p->ll_accuracy, + NUM_LL_SYMBOLS, + k_PredefRecords_LL, + seqMode)) + + seqMode = (mode >> 4) & 3; + if (seqMode == k_SeqMode_Repeat) + { if (!IS_SEQ_TABLES_WERE_SET(p)) return SZ_ERROR_DATA; } + else RINOK(FSE_Decode_SeqTable( + p->fse.of, + &in, + 5, // predefAccuracy + &p->of_accuracy, + NUM_OFFSET_SYMBOLS_MAX, + k_PredefRecords_OF, + seqMode)) + + seqMode = (mode >> 2) & 3; + if (seqMode == k_SeqMode_Repeat) + { if (!IS_SEQ_TABLES_WERE_SET(p)) return SZ_ERROR_DATA; } + else + { + RINOK(FSE_Decode_SeqTable( + p->fse.ml, + &in, + 6, // predefAccuracy + &p->ml_accuracy, + NUM_ML_SYMBOLS, + k_PredefRecords_ML, + seqMode)) + /* + #if defined(Z7_ZSTD_DEC_USE_ML_PLUS3) + // { unsigned y = 1 << 10; do + { + const unsigned accuracy = p->ml_accuracy; + if (accuracy == 0) + p->fse.ml[0] += 3; + else + #ifdef MY_CPU_64BIT + { + // alignemt (UInt64 _pad_Alignment) in fse.ml is required for that code + UInt64 *table = (UInt64 *)(void *)p->fse.ml; + const UInt64 *end = (const UInt64 *)(const void *) + ((const Byte *)(const void *)table + ((size_t)sizeof(CFseRecord) << accuracy)); + do + { + table[0] += ((UInt64)MATCH_LEN_MIN << 32) + MATCH_LEN_MIN; + table[1] += ((UInt64)MATCH_LEN_MIN << 32) + MATCH_LEN_MIN; + table += 2; + } + while (table != end); + } + #else + { + UInt32 *table = p->fse.ml; + const UInt32 *end = (const UInt32 *)(const void *) + ((const Byte *)(const void *)table + ((size_t)sizeof(CFseRecord) << accuracy)); + do + { + table[0] += MATCH_LEN_MIN; + table[1] += MATCH_LEN_MIN; + table += 2; + table[0] += MATCH_LEN_MIN; + table[1] += MATCH_LEN_MIN; + table += 2; + } + while (table != end); + } + #endif + } + // while (--y); } + #endif + */ + } + + // p->seqTables_wereSet = True; + if (in.len == 0) + return SZ_ERROR_DATA; + return Decompress_Sequences(p, + in.ptr - SEQ_SRC_OFFSET - BIT_OFFSET_DELTA_BYTES, in.len, + p->winPos + outLimit, &vars); + } +} + + + + +// inSize != 0 +// it must do similar to ZstdDec1_DecodeBlock() +static size_t ZstdDec1_NeedTempBufferForInput( + const SizeT beforeSize, const Byte * const src, const SizeT inSize) +{ + unsigned b0; + UInt32 pos; + + #ifdef Z7_ZSTD_DEC_USE_CHECK_OF_NEED_TEMP + g_numSeqs = 1 << 24; + #else + // we have at least 3 bytes before seq data: litBlockType, numSeqs, seqMode + #define MIN_BLOCK_LZ_HEADERS_SIZE 3 + if (beforeSize >= MAX_BACKWARD_DEPTH - MIN_BLOCK_LZ_HEADERS_SIZE) + return 0; + #endif + + b0 = src[0]; + + if ((b0 & k_LitBlockType_Flag_Compressed) == 0) + { + UInt32 numLits = b0 >> 3; + pos = 1; + if (b0 & 4) + { + UInt32 v; + if (inSize < 3) + return 0; + numLits >>= 1; + v = GetUi16(src + 1); + pos = 3; + if ((b0 & 8) == 0) + { + pos = 2; + v = (Byte)v; + } + numLits += v << 4; + } + if (b0 & k_LitBlockType_Flag_RLE_or_Treeless) + numLits = 1; + pos += numLits; + } + else if (inSize < 5) + return 0; + else + { + const unsigned mode4Streams = b0 & 0xc; + const unsigned numBytes = (3 * mode4Streams + 48) >> 4; + const unsigned numBits = 4 * numBytes - 6; + UInt32 cs = GetUi32(src + 1); + cs >>= numBits; + cs &= ((UInt32)16 << numBits) - 1; + if (cs == 0) + return 0; + pos = numBytes + cs; + } + + if (pos >= inSize) + return 0; + { + UInt32 numSeqs = src[pos++]; + if (numSeqs > 127) + { + UInt32 b1; + if (pos >= inSize) + return 0; + numSeqs -= 128; + b1 = src[pos++]; + if (numSeqs == 127) + { + if (pos >= inSize) + return 0; + numSeqs = (UInt32)(src[pos++]) + 127; + } + numSeqs = (numSeqs << 8) + b1; + } + #ifdef Z7_ZSTD_DEC_USE_CHECK_OF_NEED_TEMP + g_numSeqs = numSeqs; // for debug + #endif + if (numSeqs == 0) + return 0; + } + /* + if (pos >= inSize) + return 0; + pos++; + */ + // we will have one additional byte for seqMode: + if (beforeSize + pos >= MAX_BACKWARD_DEPTH - 1) + return 0; + return 1; +} + + + +// ---------- ZSTD FRAME ---------- + +#define kBlockType_Raw 0 +#define kBlockType_RLE 1 +#define kBlockType_Compressed 2 +#define kBlockType_Reserved 3 + +typedef enum +{ + // begin: states that require 4 bytes: + ZSTD2_STATE_SIGNATURE, + ZSTD2_STATE_HASH, + ZSTD2_STATE_SKIP_HEADER, + // end of states that require 4 bytes + + ZSTD2_STATE_SKIP_DATA, + ZSTD2_STATE_FRAME_HEADER, + ZSTD2_STATE_AFTER_HEADER, + ZSTD2_STATE_BLOCK, + ZSTD2_STATE_DATA, + ZSTD2_STATE_FINISHED +} EZstd2State; + + +struct CZstdDec +{ + EZstd2State frameState; + unsigned tempSize; + + Byte temp[14]; // 14 is required + + Byte descriptor; + Byte windowDescriptor; + Byte isLastBlock; + Byte blockType; + Byte isErrorState; + Byte hashError; + Byte disableHash; + Byte isCyclicMode; + + UInt32 blockSize; + UInt32 dictionaryId; + UInt32 curBlockUnpackRem; // for compressed blocks only + UInt32 inTempPos; + + UInt64 contentSize; + UInt64 contentProcessed; + CXxh64State xxh64; + + Byte *inTemp; + SizeT winBufSize_Allocated; + Byte *win_Base; + + ISzAllocPtr alloc_Small; + ISzAllocPtr alloc_Big; + + CZstdDec1 decoder; +}; + +#define ZstdDec_GET_UNPROCESSED_XXH64_SIZE(p) \ + ((unsigned)(p)->contentProcessed & (Z7_XXH64_BLOCK_SIZE - 1)) + +#define ZSTD_DEC_IS_LAST_BLOCK(p) ((p)->isLastBlock) + + +static void ZstdDec_FreeWindow(CZstdDec * const p) +{ + if (p->win_Base) + { + ISzAlloc_Free(p->alloc_Big, p->win_Base); + p->win_Base = NULL; + // p->decoder.win = NULL; + p->winBufSize_Allocated = 0; + } +} + + +CZstdDecHandle ZstdDec_Create(ISzAllocPtr alloc_Small, ISzAllocPtr alloc_Big) +{ + CZstdDec *p = (CZstdDec *)ISzAlloc_Alloc(alloc_Small, sizeof(CZstdDec)); + if (!p) + return NULL; + p->alloc_Small = alloc_Small; + p->alloc_Big = alloc_Big; + // ZstdDec_CONSTRUCT(p) + p->inTemp = NULL; + p->win_Base = NULL; + p->winBufSize_Allocated = 0; + p->disableHash = False; + ZstdDec1_Construct(&p->decoder); + return p; +} + +void ZstdDec_Destroy(CZstdDecHandle p) +{ + #ifdef SHOW_STAT + #define PRINT_STAT1(name, v) \ + printf("\n%25s = %9u", name, v); + PRINT_STAT1("g_Num_Blocks_Compressed", g_Num_Blocks_Compressed) + PRINT_STAT1("g_Num_Blocks_memcpy", g_Num_Blocks_memcpy) + PRINT_STAT1("g_Num_Wrap_memmove_Num", g_Num_Wrap_memmove_Num) + PRINT_STAT1("g_Num_Wrap_memmove_Bytes", g_Num_Wrap_memmove_Bytes) + if (g_Num_Blocks_Compressed) + { + #define PRINT_STAT(name, v) \ + printf("\n%17s = %9u, per_block = %8u", name, v, v / g_Num_Blocks_Compressed); + PRINT_STAT("g_NumSeqs", g_NumSeqs_total) + // PRINT_STAT("g_NumCopy", g_NumCopy) + PRINT_STAT("g_NumOver", g_NumOver) + PRINT_STAT("g_NumOver2", g_NumOver2) + PRINT_STAT("g_Num_Match", g_Num_Match) + PRINT_STAT("g_Num_Lits", g_Num_Lits) + PRINT_STAT("g_Num_LitsBig", g_Num_LitsBig) + PRINT_STAT("g_Num_Lit0", g_Num_Lit0) + PRINT_STAT("g_Num_Rep_0", g_Num_Rep0) + PRINT_STAT("g_Num_Rep_1", g_Num_Rep1) + PRINT_STAT("g_Num_Rep_2", g_Num_Rep2) + PRINT_STAT("g_Num_Rep_3", g_Num_Rep3) + PRINT_STAT("g_Num_Threshold_0", g_Num_Threshold_0) + PRINT_STAT("g_Num_Threshold_1", g_Num_Threshold_1) + PRINT_STAT("g_Num_Threshold_0sum", g_Num_Threshold_0sum) + PRINT_STAT("g_Num_Threshold_1sum", g_Num_Threshold_1sum) + } + printf("\n"); + #endif + + ISzAlloc_Free(p->alloc_Small, p->decoder.literalsBase); + // p->->decoder.literalsBase = NULL; + ISzAlloc_Free(p->alloc_Small, p->inTemp); + // p->inTemp = NULL; + ZstdDec_FreeWindow(p); + ISzAlloc_Free(p->alloc_Small, p); +} + + + +#define kTempBuffer_PreSize (1u << 6) +#if kTempBuffer_PreSize < MAX_BACKWARD_DEPTH + #error Stop_Compiling_Bad_kTempBuffer_PreSize +#endif + +static SRes ZstdDec_AllocateMisc(CZstdDec *p) +{ + #define k_Lit_AfterAvail (1u << 6) + #if k_Lit_AfterAvail < (COPY_CHUNK_SIZE - 1) + #error Stop_Compiling_Bad_k_Lit_AfterAvail + #endif + // return ZstdDec1_Allocate(&p->decoder, p->alloc_Small); + if (!p->decoder.literalsBase) + { + p->decoder.literalsBase = (Byte *)ISzAlloc_Alloc(p->alloc_Small, + kBlockSizeMax + k_Lit_AfterAvail); + if (!p->decoder.literalsBase) + return SZ_ERROR_MEM; + } + if (!p->inTemp) + { + // we need k_Lit_AfterAvail here for owerread from raw literals stream + p->inTemp = (Byte *)ISzAlloc_Alloc(p->alloc_Small, + kBlockSizeMax + kTempBuffer_PreSize + k_Lit_AfterAvail); + if (!p->inTemp) + return SZ_ERROR_MEM; + } + return SZ_OK; +} + + +static void ZstdDec_Init_ForNewFrame(CZstdDec *p) +{ + p->frameState = ZSTD2_STATE_SIGNATURE; + p->tempSize = 0; + + p->isErrorState = False; + p->hashError = False; + p->isCyclicMode = False; + p->contentProcessed = 0; + Xxh64State_Init(&p->xxh64); + ZstdDec1_Init(&p->decoder); +} + + +void ZstdDec_Init(CZstdDec *p) +{ + ZstdDec_Init_ForNewFrame(p); + p->decoder.winPos = 0; + memset(p->temp, 0, sizeof(p->temp)); +} + + +#define DESCRIPTOR_Get_DictionaryId_Flag(d) ((d) & 3) +#define DESCRIPTOR_FLAG_CHECKSUM (1 << 2) +#define DESCRIPTOR_FLAG_RESERVED (1 << 3) +// #define DESCRIPTOR_FLAG_UNUSED (1 << 4) +#define DESCRIPTOR_FLAG_SINGLE (1 << 5) +#define DESCRIPTOR_Get_ContentSize_Flag3(d) ((d) >> 5) +#define DESCRIPTOR_Is_ContentSize_Defined(d) (((d) & 0xe0) != 0) + + +static EZstd2State ZstdDec_UpdateState(CZstdDec * const p, const Byte b, CZstdDecInfo * const info) +{ + unsigned tempSize = p->tempSize; + p->temp[tempSize++] = b; + p->tempSize = tempSize; + + if (p->frameState == ZSTD2_STATE_BLOCK) + { + if (tempSize < 3) + return ZSTD2_STATE_BLOCK; + { + UInt32 b0 = GetUi32(p->temp); + const unsigned type = ((unsigned)b0 >> 1) & 3; + if (type == kBlockType_RLE && tempSize == 3) + return ZSTD2_STATE_BLOCK; + // info->num_Blocks_forType[type]++; + info->num_Blocks++; + if (type == kBlockType_Reserved) + { + p->isErrorState = True; // SZ_ERROR_UNSUPPORTED + return ZSTD2_STATE_BLOCK; + } + p->blockType = (Byte)type; + p->isLastBlock = (Byte)(b0 & 1); + p->inTempPos = 0; + p->tempSize = 0; + b0 >>= 3; + b0 &= 0x1fffff; + // info->num_BlockBytes_forType[type] += b0; + if (b0 == 0) + { + // empty RAW/RLE blocks are allowed in original-zstd decoder + if (type == kBlockType_Compressed) + { + p->isErrorState = True; + return ZSTD2_STATE_BLOCK; + } + if (!ZSTD_DEC_IS_LAST_BLOCK(p)) + return ZSTD2_STATE_BLOCK; + if (p->descriptor & DESCRIPTOR_FLAG_CHECKSUM) + return ZSTD2_STATE_HASH; + return ZSTD2_STATE_FINISHED; + } + p->blockSize = b0; + { + UInt32 blockLim = ZstdDec1_GET_BLOCK_SIZE_LIMIT(&p->decoder); + // compressed and uncompressed block sizes cannot be larger than min(kBlockSizeMax, window_size) + if (b0 > blockLim) + { + p->isErrorState = True; // SZ_ERROR_UNSUPPORTED; + return ZSTD2_STATE_BLOCK; + } + if (DESCRIPTOR_Is_ContentSize_Defined(p->descriptor)) + { + const UInt64 rem = p->contentSize - p->contentProcessed; + if (blockLim > rem) + blockLim = (UInt32)rem; + } + p->curBlockUnpackRem = blockLim; + // uncompressed block size cannot be larger than remain data size: + if (type != kBlockType_Compressed) + { + if (b0 > blockLim) + { + p->isErrorState = True; // SZ_ERROR_UNSUPPORTED; + return ZSTD2_STATE_BLOCK; + } + } + } + } + return ZSTD2_STATE_DATA; + } + + if ((unsigned)p->frameState < ZSTD2_STATE_SKIP_DATA) + { + UInt32 v; + if (tempSize != 4) + return p->frameState; + v = GetUi32(p->temp); + if ((unsigned)p->frameState < ZSTD2_STATE_HASH) // == ZSTD2_STATE_SIGNATURE + { + if (v == 0xfd2fb528) + { + p->tempSize = 0; + info->num_DataFrames++; + return ZSTD2_STATE_FRAME_HEADER; + } + if ((v & 0xfffffff0) == 0x184d2a50) + { + p->tempSize = 0; + info->num_SkipFrames++; + return ZSTD2_STATE_SKIP_HEADER; + } + p->isErrorState = True; + return ZSTD2_STATE_SIGNATURE; + // return ZSTD2_STATE_ERROR; // is not ZSTD stream + } + if (p->frameState == ZSTD2_STATE_HASH) + { + info->checksum_Defined = True; + info->checksum = v; + // #ifndef DISABLE_XXH_CHECK + if (!p->disableHash) + { + if (p->decoder.winPos < ZstdDec_GET_UNPROCESSED_XXH64_SIZE(p)) + { + // unexpected code failure + p->isErrorState = True; + // SZ_ERROR_FAIL; + } + else + if ((UInt32)Xxh64State_Digest(&p->xxh64, + p->decoder.win + (p->decoder.winPos - ZstdDec_GET_UNPROCESSED_XXH64_SIZE(p)), + p->contentProcessed) != v) + { + p->hashError = True; + // return ZSTD2_STATE_ERROR; // hash error + } + } + // #endif + return ZSTD2_STATE_FINISHED; + } + // (p->frameState == ZSTD2_STATE_SKIP_HEADER) + { + p->blockSize = v; + info->skipFrames_Size += v; + p->tempSize = 0; + /* we want the caller could know that there was finished frame + finished frame. So we allow the case where + we have ZSTD2_STATE_SKIP_DATA state with (blockSize == 0). + */ + // if (v == 0) return ZSTD2_STATE_SIGNATURE; + return ZSTD2_STATE_SKIP_DATA; + } + } + + // if (p->frameState == ZSTD2_STATE_FRAME_HEADER) + { + unsigned descriptor; + const Byte *h; + descriptor = p->temp[0]; + p->descriptor = (Byte)descriptor; + if (descriptor & DESCRIPTOR_FLAG_RESERVED) // reserved bit + { + p->isErrorState = True; + return ZSTD2_STATE_FRAME_HEADER; + // return ZSTD2_STATE_ERROR; + } + { + const unsigned n = DESCRIPTOR_Get_ContentSize_Flag3(descriptor); + // tempSize -= 1 + ((1u << (n >> 1)) | ((n + 1) & 1)); + tempSize -= (0x9a563422u >> (n * 4)) & 0xf; + } + if (tempSize != (4u >> (3 - DESCRIPTOR_Get_DictionaryId_Flag(descriptor)))) + return ZSTD2_STATE_FRAME_HEADER; + + info->descriptor_OR = (Byte)(info->descriptor_OR | descriptor); + info->descriptor_NOT_OR = (Byte)(info->descriptor_NOT_OR | ~descriptor); + + h = &p->temp[1]; + { + Byte w = 0; + if ((descriptor & DESCRIPTOR_FLAG_SINGLE) == 0) + { + w = *h++; + if (info->windowDescriptor_MAX < w) + info->windowDescriptor_MAX = w; + // info->are_WindowDescriptors = True; + // info->num_WindowDescriptors++; + } + else + { + // info->are_SingleSegments = True; + // info->num_SingleSegments++; + } + p->windowDescriptor = w; + } + { + unsigned n = DESCRIPTOR_Get_DictionaryId_Flag(descriptor); + UInt32 d = 0; + if (n) + { + n = 1u << (n - 1); + d = GetUi32(h) & ((UInt32)(Int32)-1 >> (32 - 8u * n)); + h += n; + } + p->dictionaryId = d; + // info->dictionaryId_Cur = d; + if (d != 0) + { + if (info->dictionaryId == 0) + info->dictionaryId = d; + else if (info->dictionaryId != d) + info->are_DictionaryId_Different = True; + } + } + { + unsigned n = DESCRIPTOR_Get_ContentSize_Flag3(descriptor); + UInt64 v = 0; + if (n) + { + n >>= 1; + if (n == 1) + v = 256; + v += GetUi64(h) & ((UInt64)(Int64)-1 >> (64 - (8u << n))); + // info->are_ContentSize_Known = True; + // info->num_Frames_with_ContentSize++; + if (info->contentSize_MAX < v) + info->contentSize_MAX = v; + info->contentSize_Total += v; + } + else + { + info->are_ContentSize_Unknown = True; + // info->num_Frames_without_ContentSize++; + } + p->contentSize = v; + } + // if ((size_t)(h - p->temp) != headerSize) return ZSTD2_STATE_ERROR; // it's unexpected internal code failure + p->tempSize = 0; + + info->checksum_Defined = False; + /* + if (descriptor & DESCRIPTOR_FLAG_CHECKSUM) + info->are_Checksums = True; + else + info->are_Non_Checksums = True; + */ + + return ZSTD2_STATE_AFTER_HEADER; // ZSTD2_STATE_BLOCK; + } +} + + +static void ZstdDec_Update_XXH(CZstdDec * const p, size_t xxh64_winPos) +{ + /* + #ifdef DISABLE_XXH_CHECK + UNUSED_VAR(data) + #else + */ + if (!p->disableHash && (p->descriptor & DESCRIPTOR_FLAG_CHECKSUM)) + { + // const size_t pos = p->xxh64_winPos; + const size_t size = (p->decoder.winPos - xxh64_winPos) & ~(size_t)31; + if (size) + { + // p->xxh64_winPos = pos + size; + Xxh64State_UpdateBlocks(&p->xxh64, + p->decoder.win + xxh64_winPos, + p->decoder.win + xxh64_winPos + size); + } + } +} + + +/* +in: + (winLimit) : is relaxed limit, where this function is allowed to stop writing of decoded data (if possible). + - this function uses (winLimit) for RAW/RLE blocks only, + because this function can decode single RAW/RLE block in several different calls. + - this function DOESN'T use (winLimit) for Compressed blocks, + because this function decodes full compressed block in single call. + (CZstdDec1::winPos <= winLimit) + (winLimit <= CZstdDec1::cycSize). + Note: if (ds->outBuf_fromCaller) mode is used, then + { + (strong_limit) is stored in CZstdDec1::cycSize. + So (winLimit) is more strong than (strong_limit). + } + +exit: + Note: (CZstdDecState::winPos) will be set by caller after exit of this function. + + This function can exit for any of these conditions: + - (frameState == ZSTD2_STATE_AFTER_HEADER) + - (frameState == ZSTD2_STATE_FINISHED) : frame was finished : (status == ZSTD_STATUS_FINISHED_FRAME) is set + - finished non-empty non-last block. So (CZstdDec1::winPos_atExit != winPos_atFuncStart). + - ZSTD_STATUS_NEEDS_MORE_INPUT in src + - (CZstdDec1::winPos) have reached (winLimit) in non-finished RAW/RLE block + + This function decodes no more than one non-empty block. + So it fulfills the condition at exit: + (CZstdDec1::winPos_atExit - winPos_atFuncStart <= block_size_max) + Note: (winPos_atExit > winLimit) is possible in some cases after compressed block decoding. + + if (ds->outBuf_fromCaller) mode (useAdditionalWinLimit medo) + { + then this function uses additional strong limit from (CZstdDec1::cycSize). + So this function will not write any data after (CZstdDec1::cycSize) + And it fulfills the condition at exit: + (CZstdDec1::winPos_atExit <= CZstdDec1::cycSize) + } +*/ +static SRes ZstdDec_DecodeBlock(CZstdDec * const p, CZstdDecState * const ds, + SizeT winLimitAdd) +{ + const Byte *src = ds->inBuf; + SizeT * const srcLen = &ds->inPos; + const SizeT inSize = ds->inLim; + // const int useAdditionalWinLimit = ds->outBuf_fromCaller ? 1 : 0; + enum_ZstdStatus * const status = &ds->status; + CZstdDecInfo * const info = &ds->info; + SizeT winLimit; + + const SizeT winPos_atFuncStart = p->decoder.winPos; + src += *srcLen; + *status = ZSTD_STATUS_NOT_SPECIFIED; + + // finishMode = ZSTD_FINISH_ANY; + if (ds->outSize_Defined) + { + if (ds->outSize < ds->outProcessed) + { + // p->isAfterSizeMode = 2; // we have extra bytes already + *status = ZSTD_STATUS_OUT_REACHED; + return SZ_OK; + // size = 0; + } + else + { + // p->outSize >= p->outProcessed + const UInt64 rem = ds->outSize - ds->outProcessed; + /* + if (rem == 0) + p->isAfterSizeMode = 1; // we have reached exact required size + */ + if (winLimitAdd >= rem) + { + winLimitAdd = (SizeT)rem; + // if (p->finishMode) finishMode = ZSTD_FINISH_END; + } + } + } + + winLimit = p->decoder.winPos + winLimitAdd; + // (p->decoder.winPos <= winLimit) + + // while (p->frameState != ZSTD2_STATE_ERROR) + while (!p->isErrorState) + { + SizeT inCur = inSize - *srcLen; + + if (p->frameState == ZSTD2_STATE_DATA) + { + /* (p->decoder.winPos == winPos_atFuncStart) is expected, + because this function doesn't start new block. + if it have finished some non-empty block in this call. */ + if (p->decoder.winPos != winPos_atFuncStart) + return SZ_ERROR_FAIL; // it's unexpected + + /* + if (p->decoder.winPos > winLimit) + { + // we can be here, if in this function call + // - we have extracted non-empty compressed block, and (winPos > winLimit) after that. + // - we have started new block decoding after that. + // It's unexpected case, because we exit after non-empty non-last block. + *status = (inSize == *srcLen) ? + ZSTD_STATUS_NEEDS_MORE_INPUT : + ZSTD_STATUS_NOT_FINISHED; + return SZ_OK; + } + */ + // p->decoder.winPos <= winLimit + + if (p->blockType != kBlockType_Compressed) + { + // it's RLE or RAW block. + // p->BlockSize != 0_ + // winLimit <= p->decoder.cycSize + /* So here we use more strong (winLimit), even for + (ds->outBuf_fromCaller) mode. */ + SizeT outCur = winLimit - p->decoder.winPos; + { + const UInt32 rem = p->blockSize; + if (outCur > rem) + outCur = rem; + } + if (p->blockType == kBlockType_Raw) + { + if (outCur > inCur) + outCur = inCur; + /* output buffer is better aligned for XXH code. + So we use hash for output buffer data */ + // ZstdDec_Update_XXH(p, src, outCur); // for debug: + memcpy(p->decoder.win + p->decoder.winPos, src, outCur); + src += outCur; + *srcLen += outCur; + } + else // kBlockType_RLE + { + #define RLE_BYTE_INDEX_IN_temp 3 + memset(p->decoder.win + p->decoder.winPos, + p->temp[RLE_BYTE_INDEX_IN_temp], outCur); + } + { + const SizeT xxh64_winPos = p->decoder.winPos - ZstdDec_GET_UNPROCESSED_XXH64_SIZE(p); + p->decoder.winPos += outCur; + p->contentProcessed += outCur; + ZstdDec_Update_XXH(p, xxh64_winPos); + } + // ds->winPos = p->decoder.winPos; // the caller does it instead. for debug: + UPDATE_TOTAL_OUT(&p->decoder, outCur) + ds->outProcessed += outCur; + if (p->blockSize -= (UInt32)outCur) + { + /* + if (ds->outSize_Defined) + { + if (ds->outSize <= ds->outProcessed) ds->isAfterSizeMode = (enum_ZstdStatus) + (ds->outSize == ds->outProcessed ? 1u: 2u); + } + */ + *status = (enum_ZstdStatus) + (ds->outSize_Defined && ds->outSize <= ds->outProcessed ? + ZSTD_STATUS_OUT_REACHED : (p->blockType == kBlockType_Raw && inSize == *srcLen) ? + ZSTD_STATUS_NEEDS_MORE_INPUT : + ZSTD_STATUS_NOT_FINISHED); + return SZ_OK; + } + } + else // kBlockType_Compressed + { + // p->blockSize != 0 + // (uncompressed_size_of_block == 0) is allowed + // (p->curBlockUnpackRem == 0) is allowed + /* + if (p->decoder.winPos >= winLimit) + { + if (p->decoder.winPos != winPos_atFuncStart) + { + // it's unexpected case + // We already have some data in finished blocks in this function call. + // So we don't decompress new block after (>=winLimit), + // even if it's empty block. + *status = (inSize == *srcLen) ? + ZSTD_STATUS_NEEDS_MORE_INPUT : + ZSTD_STATUS_NOT_FINISHED; + return SZ_OK; + } + // (p->decoder.winPos == winLimit == winPos_atFuncStart) + // we will decode current block, because that current + // block can be empty block and we want to make some visible + // change of (src) stream after function start. + } + */ + /* + if (ds->outSize_Defined && ds->outSize < ds->outProcessed) + { + // we don't want to start new block, if we have more extra decoded bytes already + *status = ZSTD_STATUS_OUT_REACHED; + return SZ_OK; + } + */ + { + const Byte *comprStream; + size_t afterAvail; + UInt32 inTempPos = p->inTempPos; + const UInt32 rem = p->blockSize - inTempPos; + // rem != 0 + if (inTempPos != 0 // (inTemp) buffer already contains some input data + || inCur < rem // available input data size is smaller than compressed block size + || ZstdDec1_NeedTempBufferForInput(*srcLen, src, rem)) + { + if (inCur > rem) + inCur = rem; + if (inCur) + { + STAT_INC(g_Num_Blocks_memcpy) + // we clear data for backward lookahead reading + if (inTempPos == 0) + memset(p->inTemp + kTempBuffer_PreSize - MAX_BACKWARD_DEPTH, 0, MAX_BACKWARD_DEPTH); + // { unsigned y = 0; for(;y < 1000; y++) + memcpy(p->inTemp + inTempPos + kTempBuffer_PreSize, src, inCur); + // } + src += inCur; + *srcLen += inCur; + inTempPos += (UInt32)inCur; + p->inTempPos = inTempPos; + } + if (inTempPos != p->blockSize) + { + *status = ZSTD_STATUS_NEEDS_MORE_INPUT; + return SZ_OK; + } + #if COPY_CHUNK_SIZE > 1 + memset(p->inTemp + kTempBuffer_PreSize + inTempPos, 0, COPY_CHUNK_SIZE); + #endif + comprStream = p->inTemp + kTempBuffer_PreSize; + afterAvail = k_Lit_AfterAvail; + // we don't want to read non-initialized data or junk in CopyMatch(): + } + else + { + // inCur >= rem + // we use direct decoding from (src) buffer: + afterAvail = inCur - rem; + comprStream = src; + src += rem; + *srcLen += rem; + } + + #ifdef Z7_ZSTD_DEC_USE_CHECK_OF_NEED_TEMP + ZstdDec1_NeedTempBufferForInput(*srcLen, comprStream, p->blockSize); + #endif + // printf("\nblockSize=%u", p->blockSize); + // printf("%x\n", (unsigned)p->contentProcessed); + STAT_INC(g_Num_Blocks_Compressed) + { + SRes sres; + const size_t winPos = p->decoder.winPos; + /* + if ( useAdditionalWinLimit), we use strong unpack limit: smallest from + - limit from stream : (curBlockUnpackRem) + - limit from caller : (cycSize - winPos) + if (!useAdditionalWinLimit), we use only relaxed limit: + - limit from stream : (curBlockUnpackRem) + */ + SizeT outLimit = p->curBlockUnpackRem; + if (ds->outBuf_fromCaller) + // if (useAdditionalWinLimit) + { + const size_t limit = p->decoder.cycSize - winPos; + if (outLimit > limit) + outLimit = limit; + } + sres = ZstdDec1_DecodeBlock(&p->decoder, + comprStream, p->blockSize, afterAvail, outLimit); + // ds->winPos = p->decoder.winPos; // the caller does it instead. for debug: + if (sres) + { + p->isErrorState = True; + return sres; + } + { + const SizeT xxh64_winPos = winPos - ZstdDec_GET_UNPROCESSED_XXH64_SIZE(p); + const size_t num = p->decoder.winPos - winPos; + ds->outProcessed += num; + p->contentProcessed += num; + ZstdDec_Update_XXH(p, xxh64_winPos); + } + } + // printf("\nwinPos=%x", (int)(unsigned)p->decoder.winPos); + } + } + + /* + if (ds->outSize_Defined) + { + if (ds->outSize <= ds->outProcessed) ds->isAfterSizeMode = (enum_ZstdStatus) + (ds->outSize == ds->outProcessed ? 1u: 2u); + } + */ + + if (!ZSTD_DEC_IS_LAST_BLOCK(p)) + { + p->frameState = ZSTD2_STATE_BLOCK; + if (ds->outSize_Defined && ds->outSize < ds->outProcessed) + { + *status = ZSTD_STATUS_OUT_REACHED; + return SZ_OK; + } + // we exit only if (winPos) was changed in this function call: + if (p->decoder.winPos != winPos_atFuncStart) + { + // decoded block was not empty. So we exit: + *status = (enum_ZstdStatus)( + (inSize == *srcLen) ? + ZSTD_STATUS_NEEDS_MORE_INPUT : + ZSTD_STATUS_NOT_FINISHED); + return SZ_OK; + } + // (p->decoder.winPos == winPos_atFuncStart) + // so current decoded block was empty. + // we will try to decode more blocks in this function. + continue; + } + + // decoded block was last in frame + if (p->descriptor & DESCRIPTOR_FLAG_CHECKSUM) + { + p->frameState = ZSTD2_STATE_HASH; + if (ds->outSize_Defined && ds->outSize < ds->outProcessed) + { + *status = ZSTD_STATUS_OUT_REACHED; + return SZ_OK; // disable if want to + /* We want to get same return codes for any input buffer sizes. + We want to get faster ZSTD_STATUS_OUT_REACHED status. + So we exit with ZSTD_STATUS_OUT_REACHED here, + instead of ZSTD2_STATE_HASH and ZSTD2_STATE_FINISHED processing. + that depends from input buffer size and that can set + ZSTD_STATUS_NEEDS_MORE_INPUT or return SZ_ERROR_DATA or SZ_ERROR_CRC. + */ + } + } + else + { + /* ZSTD2_STATE_FINISHED proccesing doesn't depend from input buffer */ + p->frameState = ZSTD2_STATE_FINISHED; + } + /* + p->frameState = (p->descriptor & DESCRIPTOR_FLAG_CHECKSUM) ? + ZSTD2_STATE_HASH : + ZSTD2_STATE_FINISHED; + */ + /* it's required to process ZSTD2_STATE_FINISHED state in this function call, + because we must check contentSize and hashError in ZSTD2_STATE_FINISHED code, + while the caller can reinit full state for ZSTD2_STATE_FINISHED + So we can't exit from function here. */ + continue; + } + + if (p->frameState == ZSTD2_STATE_FINISHED) + { + *status = ZSTD_STATUS_FINISHED_FRAME; + if (DESCRIPTOR_Is_ContentSize_Defined(p->descriptor) + && p->contentSize != p->contentProcessed) + return SZ_ERROR_DATA; + if (p->hashError) // for debug + return SZ_ERROR_CRC; + return SZ_OK; + // p->frameState = ZSTD2_STATE_SIGNATURE; + // continue; + } + + if (p->frameState == ZSTD2_STATE_AFTER_HEADER) + return SZ_OK; // we need memory allocation for that state + + if (p->frameState == ZSTD2_STATE_SKIP_DATA) + { + UInt32 blockSize = p->blockSize; + // (blockSize == 0) is possible + if (inCur > blockSize) + inCur = blockSize; + src += inCur; + *srcLen += inCur; + blockSize -= (UInt32)inCur; + p->blockSize = blockSize; + if (blockSize == 0) + { + p->frameState = ZSTD2_STATE_SIGNATURE; + // continue; // for debug: we can continue without return to caller. + // we notify the caller that skip frame was finished: + *status = ZSTD_STATUS_FINISHED_FRAME; + return SZ_OK; + } + // blockSize != 0 + // (inCur) was smaller than previous value of p->blockSize. + // (inSize == *srcLen) now + *status = ZSTD_STATUS_NEEDS_MORE_INPUT; + return SZ_OK; + } + + if (inCur == 0) + { + *status = ZSTD_STATUS_NEEDS_MORE_INPUT; + return SZ_OK; + } + + { + (*srcLen)++; + p->frameState = ZstdDec_UpdateState(p, *src++, info); + } + } + + *status = ZSTD_STATUS_NOT_SPECIFIED; + p->isErrorState = True; + // p->frameState = ZSTD2_STATE_ERROR; + // if (p->frameState = ZSTD2_STATE_SIGNATURE) return SZ_ERROR_NO_ARCHIVE + return SZ_ERROR_DATA; +} + + + + +SRes ZstdDec_Decode(CZstdDecHandle dec, CZstdDecState *p) +{ + p->needWrite_Size = 0; + p->status = ZSTD_STATUS_NOT_SPECIFIED; + dec->disableHash = p->disableHash; + + if (p->outBuf_fromCaller) + { + dec->decoder.win = p->outBuf_fromCaller; + dec->decoder.cycSize = p->outBufSize_fromCaller; + } + + // p->winPos = dec->decoder.winPos; + + for (;;) + { + SizeT winPos, size; + // SizeT outProcessed; + SRes res; + + if (p->wrPos > dec->decoder.winPos) + return SZ_ERROR_FAIL; + + if (dec->frameState == ZSTD2_STATE_FINISHED) + { + if (!p->outBuf_fromCaller) + { + // we need to set positions to zero for new frame. + if (p->wrPos != dec->decoder.winPos) + { + /* We have already asked the caller to flush all data + with (p->needWrite_Size) and (ZSTD_STATUS_FINISHED_FRAME) status. + So it's unexpected case */ + // p->winPos = dec->decoder.winPos; + // p->needWrite_Size = dec->decoder.winPos - p->wrPos; // flush size asking + // return SZ_OK; // ask to flush again + return SZ_ERROR_FAIL; + } + // (p->wrPos == dec->decoder.winPos), and we wrap to zero: + dec->decoder.winPos = 0; + p->winPos = 0; + p->wrPos = 0; + } + ZstdDec_Init_ForNewFrame(dec); + // continue; + } + + winPos = dec->decoder.winPos; + { + SizeT next = dec->decoder.cycSize; + /* cycSize == 0, if no buffer was allocated still, + or, if (outBuf_fromCaller) mode and (outBufSize_fromCaller == 0) */ + if (!p->outBuf_fromCaller + && next + && next <= winPos + && dec->isCyclicMode) + { + // (0 < decoder.cycSize <= winPos) in isCyclicMode. + // so we need to wrap (winPos) and (wrPos) over (cycSize). + const size_t delta = next; + // (delta) is how many bytes we remove from buffer. + /* + // we don't need data older than last (cycSize) bytes. + size_t delta = winPos - next; // num bytes after (cycSize) + if (delta <= next) // it's expected case + delta = next; + // delta == Max(cycSize, winPos - cycSize) + */ + if (p->wrPos < delta) + { + // (wrPos < decoder.cycSize) + // We have asked already the caller to flush required data + // p->status = ZSTD_STATUS_NOT_SPECIFIED; + // p->winPos = winPos; + // p->needWrite_Size = delta - p->wrPos; // flush size asking + // return SZ_OK; // ask to flush again + return SZ_ERROR_FAIL; + } + // p->wrPos >= decoder.cycSize + // we move extra data after (decoder.cycSize) to start of cyclic buffer: + winPos -= delta; + if (winPos) + { + if (winPos >= delta) + return SZ_ERROR_FAIL; + memmove(dec->decoder.win, dec->decoder.win + delta, winPos); + // printf("\nmemmove processed=%8x winPos=%8x\n", (unsigned)p->outProcessed, (unsigned)dec->decoder.winPos); + STAT_INC(g_Num_Wrap_memmove_Num) + STAT_UPDATE(g_Num_Wrap_memmove_Bytes += (unsigned)winPos;) + } + dec->decoder.winPos = winPos; + p->winPos = winPos; + p->wrPos -= delta; + // dec->xxh64_winPos -= delta; + + // (winPos < delta) + #ifdef Z7_STD_DEC_USE_AFTER_CYC_BUF + /* we set the data after cycSize, because + we don't want to read non-initialized data or junk in CopyMatch(). */ + memset(dec->decoder.win + next, 0, COPY_CHUNK_SIZE); + #endif + + /* + if (winPos == next) + { + if (winPos != p->wrPos) + { + // we already requested before to flush full data for that case. + // but we give the caller a second chance to flush data: + p->needWrite_Size = winPos - p->wrPos; + return SZ_OK; + } + // (decoder.cycSize == winPos == p->wrPos) + // so we do second wrapping to zero: + winPos = 0; + dec->decoder.winPos = 0; + p->winPos = 0; + p->wrPos = 0; + } + */ + // (winPos < next) + } + + if (winPos > next) + return SZ_ERROR_FAIL; // it's unexpected case + /* + if (!outBuf_fromCaller && isCyclicMode && cycSize != 0) + then (winPos < cycSize) + else (winPos <= cycSize) + */ + if (!p->outBuf_fromCaller) + { + // that code is optional. We try to optimize write chunk sizes. + /* (next2) is expected next write position in the caller, + if the caller writes by kBlockSizeMax chunks. + */ + /* + const size_t next2 = (winPos + kBlockSizeMax) & (kBlockSizeMax - 1); + if (winPos < next2 && next2 < next) + next = next2; + */ + } + size = next - winPos; + } + + // note: ZstdDec_DecodeBlock() uses (winLimit = winPos + size) only for RLE and RAW blocks + res = ZstdDec_DecodeBlock(dec, p, size); + /* + after one block decoding: + if (!outBuf_fromCaller && isCyclicMode && cycSize != 0) + then (winPos < cycSize + max_block_size) + else (winPos <= cycSize) + */ + + if (!p->outBuf_fromCaller) + p->win = dec->decoder.win; + p->winPos = dec->decoder.winPos; + + // outProcessed = dec->decoder.winPos - winPos; + // p->outProcessed += outProcessed; + + if (res != SZ_OK) + return res; + + if (dec->frameState != ZSTD2_STATE_AFTER_HEADER) + { + if (p->outBuf_fromCaller) + return SZ_OK; + { + // !p->outBuf_fromCaller + /* + if (ZSTD_STATUS_FINISHED_FRAME), we request full flushing here because + 1) it's simpler to work with allocation and extracting of next frame, + 2) it's better to start writing to next new frame with aligned memory + for faster xxh 64-bit reads. + */ + size_t end = dec->decoder.winPos; // end pos for all data flushing + if (p->status != ZSTD_STATUS_FINISHED_FRAME) + { + // we will request flush here only for cases when wrap in cyclic buffer can be required in next call. + if (!dec->isCyclicMode) + return SZ_OK; + // isCyclicMode + { + const size_t delta = dec->decoder.cycSize; + if (end < delta) + return SZ_OK; // (winPos < cycSize). no need for flush + // cycSize <= winPos + // So we ask the caller to flush of (cycSize - wrPos) bytes, + // and then we will wrap cylicBuffer in next call + end = delta; + } + } + p->needWrite_Size = end - p->wrPos; + } + return SZ_OK; + } + + // ZSTD2_STATE_AFTER_HEADER + { + BoolInt useCyclic = False; + size_t cycSize; + + // p->status = ZSTD_STATUS_NOT_FINISHED; + if (dec->dictionaryId != 0) + { + /* actually we can try to decode some data, + because it's possible that some data doesn't use dictionary */ + // p->status = ZSTD_STATUS_NOT_SPECIFIED; + return SZ_ERROR_UNSUPPORTED; + } + + { + UInt64 winSize = dec->contentSize; + UInt64 winSize_Allocate = winSize; + const unsigned descriptor = dec->descriptor; + + if ((descriptor & DESCRIPTOR_FLAG_SINGLE) == 0) + { + const Byte wd = dec->windowDescriptor; + winSize = (UInt64)(8 + (wd & 7)) << ((wd >> 3) + 10 - 3); + if (!DESCRIPTOR_Is_ContentSize_Defined(descriptor) + || winSize_Allocate > winSize) + { + winSize_Allocate = winSize; + useCyclic = True; + } + } + /* + else + { + if (p->info.singleSegment_ContentSize_MAX < winSize) + p->info.singleSegment_ContentSize_MAX = winSize; + // p->info.num_SingleSegments++; + } + */ + if (p->info.windowSize_MAX < winSize) + p->info.windowSize_MAX = winSize; + if (p->info.windowSize_Allocate_MAX < winSize_Allocate) + p->info.windowSize_Allocate_MAX = winSize_Allocate; + /* + winSize_Allocate is MIN(content_size, window_size_from_descriptor). + Wven if (content_size < (window_size_from_descriptor)) + original-zstd still uses (window_size_from_descriptor) to check that decoding is allowed. + We try to follow original-zstd, and here we check (winSize) instead of (winSize_Allocate)) + */ + if ( + // winSize_Allocate // it's relaxed check + winSize // it's more strict check to be compatible with original-zstd + > ((UInt64)1 << MAX_WINDOW_SIZE_LOG)) + return SZ_ERROR_UNSUPPORTED; // SZ_ERROR_MEM + cycSize = (size_t)winSize_Allocate; + if (cycSize != winSize_Allocate) + return SZ_ERROR_MEM; + // cycSize <= winSize + /* later we will use (CZstdDec1::winSize) to check match offsets and check block sizes. + if (there is window descriptor) + { + We will check block size with (window_size_from_descriptor) instead of (winSize_Allocate). + Does original-zstd do it that way also? + } + Here we must reduce full real 64-bit (winSize) to size_t for (CZstdDec1::winSize). + Also we don't want too big values for (CZstdDec1::winSize). + our (CZstdDec1::winSize) will meet the condition: + (CZstdDec1::winSize < kBlockSizeMax || CZstdDec1::winSize <= cycSize). + */ + dec->decoder.winSize = (winSize < kBlockSizeMax) ? (size_t)winSize: cycSize; + // note: (CZstdDec1::winSize > cycSize) is possible, if (!useCyclic) + } + + RINOK(ZstdDec_AllocateMisc(dec)) + + if (p->outBuf_fromCaller) + dec->isCyclicMode = False; + else + { + size_t d = cycSize; + + if (dec->decoder.winPos != p->wrPos) + return SZ_ERROR_FAIL; + + dec->decoder.winPos = 0; + p->wrPos = 0; + p->winPos = dec->decoder.winPos; + + /* + const size_t needWrite = dec->decoder.winPos - p->wrPos; + if (!needWrite) + { + dec->decoder.winPos = 0; + p->wrPos = 0; + p->winPos = dec->decoder.winPos; + } + */ + /* if (!useCyclic) we allocate only cycSize = ContentSize. + But if we want to support the case where new frame starts with winPos != 0, + then we will wrap over zero, and we still need + to set (useCyclic) and allocate additional buffer spaces. + Now we don't allow new frame starting with (winPos != 0). + so (dec->decoder->winPos == 0) + can use (!useCyclic) with reduced buffer sizes. + */ + /* + if (dec->decoder->winPos != 0) + useCyclic = True; + */ + + if (useCyclic) + { + /* cyclyc buffer size must be at least (COPY_CHUNK_SIZE - 1) bytes + larger than window size, because CopyMatch() can write additional + (COPY_CHUNK_SIZE - 1) bytes and overwrite oldests data in cyclyc buffer. + But for performance reasons we align (cycSize) for (kBlockSizeMax). + also we must provide (cycSize >= max_decoded_data_after_cycSize), + because after data move wrapping over zero we must provide (winPos < cycSize). + */ + const size_t alignSize = kBlockSizeMax; + /* here we add (1 << 7) instead of (COPY_CHUNK_SIZE - 1), because + we want to get same (cycSize) for different COPY_CHUNK_SIZE values. */ + // cycSize += (COPY_CHUNK_SIZE - 1) + (alignSize - 1); // for debug : we can get smallest (cycSize) + cycSize += (1 << 7) + alignSize; + cycSize &= ~(size_t)(alignSize - 1); + // cycSize must be aligned for 32, because xxh requires 32-bytes blocks. + // cycSize += 12345; // for debug + // cycSize += 1 << 10; // for debug + // cycSize += 32; // for debug + // cycSize += kBlockSizeMax; // for debug + if (cycSize < d) + return SZ_ERROR_MEM; + /* + in cyclic buffer mode we allow to decode one additional block + that exceeds (cycSize). + So we must allocate additional (kBlockSizeMax) bytes after (cycSize). + if defined(Z7_STD_DEC_USE_AFTER_CYC_BUF) + { + we can read (COPY_CHUNK_SIZE - 1) bytes after (cycSize) + but we aready allocate additional kBlockSizeMax that + is larger than COPY_CHUNK_SIZE. + So we don't need additional space of COPY_CHUNK_SIZE after (cycSize). + } + */ + /* + #ifdef Z7_STD_DEC_USE_AFTER_CYC_BUF + d = cycSize + (1 << 7); // we must add at least (COPY_CHUNK_SIZE - 1) + #endif + */ + d = cycSize + kBlockSizeMax; + if (d < cycSize) + return SZ_ERROR_MEM; + } + + { + const size_t kMinWinAllocSize = 1 << 12; + if (d < kMinWinAllocSize) + d = kMinWinAllocSize; + } + + if (d > dec->winBufSize_Allocated) + { + /* + if (needWrite) + { + p->needWrite_Size = needWrite; + return SZ_OK; + // return SZ_ERROR_FAIL; + } + */ + + if (dec->winBufSize_Allocated != 0) + { + const size_t k_extra = (useCyclic || d >= (1u << 20)) ? + 2 * kBlockSizeMax : 0; + unsigned i = useCyclic ? 17 : 12; + for (; i < sizeof(size_t) * 8; i++) + { + const size_t d2 = ((size_t)1 << i) + k_extra; + if (d2 >= d) + { + d = d2; + break; + } + } + } + // RINOK(ZstdDec_AllocateWindow(dec, d)) + ZstdDec_FreeWindow(dec); + dec->win_Base = (Byte *)ISzAlloc_Alloc(dec->alloc_Big, d); + if (!dec->win_Base) + return SZ_ERROR_MEM; + dec->decoder.win = dec->win_Base; + dec->winBufSize_Allocated = d; + } + /* + else + { + // for non-cyclycMode we want flush data, and set winPos = 0 + if (needWrite) + { + if (!useCyclic || dec->decoder.winPos >= cycSize) + { + p->needWrite_Size = needWrite; + return SZ_OK; + // return SZ_ERROR_FAIL; + } + } + } + */ + + dec->decoder.cycSize = cycSize; + p->win = dec->decoder.win; + // p->cycSize = dec->decoder.cycSize; + dec->isCyclicMode = (Byte)useCyclic; + } // (!p->outBuf_fromCaller) end + + // p->winPos = dec->decoder.winPos; + dec->frameState = ZSTD2_STATE_BLOCK; + // continue; + } // ZSTD2_STATE_AFTER_HEADER end + } +} + + +void ZstdDec_GetResInfo(const CZstdDec *dec, + const CZstdDecState *p, + SRes res, + CZstdDecResInfo *stat) +{ + // ZstdDecInfo_CLEAR(stat); + stat->extraSize = 0; + stat->is_NonFinishedFrame = False; + if (dec->frameState != ZSTD2_STATE_FINISHED) + { + if (dec->frameState == ZSTD2_STATE_SIGNATURE) + { + stat->extraSize = (Byte)dec->tempSize; + if (ZstdDecInfo_GET_NUM_FRAMES(&p->info) == 0) + res = SZ_ERROR_NO_ARCHIVE; + } + else + { + stat->is_NonFinishedFrame = True; + if (res == SZ_OK && p->status == ZSTD_STATUS_NEEDS_MORE_INPUT) + res = SZ_ERROR_INPUT_EOF; + } + } + stat->decode_SRes = res; +} + + +size_t ZstdDec_ReadUnusedFromInBuf( + CZstdDecHandle dec, + size_t afterDecoding_tempPos, + void *data, size_t size) +{ + size_t processed = 0; + if (dec->frameState == ZSTD2_STATE_SIGNATURE) + { + Byte *dest = (Byte *)data; + const size_t tempSize = dec->tempSize; + while (afterDecoding_tempPos < tempSize) + { + if (size == 0) + break; + size--; + *dest++ = dec->temp[afterDecoding_tempPos++]; + processed++; + } + } + return processed; +} + + +void ZstdDecState_Clear(CZstdDecState *p) +{ + memset(p, 0 , sizeof(*p)); +} diff --git a/C/ZstdDec.h b/C/ZstdDec.h new file mode 100644 index 0000000..cd26131 --- /dev/null +++ b/C/ZstdDec.h @@ -0,0 +1,173 @@ +/* ZstdDec.h -- Zstd Decoder interfaces +2024-01-21 : Igor Pavlov : Public domain */ + +#ifndef ZIP7_INC_ZSTD_DEC_H +#define ZIP7_INC_ZSTD_DEC_H + +EXTERN_C_BEGIN + +typedef struct CZstdDec CZstdDec; +typedef CZstdDec * CZstdDecHandle; + +CZstdDecHandle ZstdDec_Create(ISzAllocPtr alloc_Small, ISzAllocPtr alloc_Big); +void ZstdDec_Destroy(CZstdDecHandle p); + +typedef enum +{ + ZSTD_STATUS_NOT_SPECIFIED, /* use main error code instead */ + ZSTD_STATUS_FINISHED_FRAME, /* data frame or skip frame was finished */ + ZSTD_STATUS_NOT_FINISHED, /* just finished non-empty block or unfinished RAW/RLE block */ + ZSTD_STATUS_NEEDS_MORE_INPUT, /* the callee needs more input bytes. It has more priority over ZSTD_STATUS_NOT_FINISHED */ + ZSTD_STATUS_OUT_REACHED /* is not finihed frame and ((outProcessed > outSize) || (outProcessed == outSize && unfinished RAW/RLE block) */ +} enum_ZstdStatus_Dummy; + +#define ZstdDecState_DOES_NEED_MORE_INPUT_OR_FINISHED_FRAME(p) \ + ((p)->status & ZSTD_STATUS_FINISHED_FRAME) +/* + ((p)->status == ZSTD_STATUS_NEEDS_MORE_INPUT || \ + (p)->status == ZSTD_STATUS_FINISHED_FRAME) +*/ + +typedef Byte enum_ZstdStatus; + + +void ZstdDec_Init(CZstdDecHandle p); + +typedef struct +{ + UInt64 num_Blocks; + Byte descriptor_OR; + Byte descriptor_NOT_OR; + Byte are_ContentSize_Unknown; + Byte windowDescriptor_MAX; + + // Byte are_ContentSize_Known; + // Byte are_SingleSegments; + // Byte are_WindowDescriptors; + Byte checksum_Defined; + // Byte are_Checksums; + // Byte are_Non_Checksums; + + // Byte are_DictionaryId; + Byte are_DictionaryId_Different; + + // Byte reserved[3]; + + UInt32 checksum; // checksum of last data frame + /// UInt32 dictionaryId_Cur; + UInt32 dictionaryId; // if there are non-zero dictionary IDs, then it's first dictionaryId + + UInt64 num_DataFrames; + UInt64 num_SkipFrames; + UInt64 skipFrames_Size; + UInt64 contentSize_Total; + UInt64 contentSize_MAX; + // UInt64 num_Checksums; + // UInt64 num_Non_Checksums; // frames without checksum + // UInt64 num_WindowDescriptors; + // UInt64 num_SingleSegments; + // UInt64 num_Frames_with_ContentSize; + // UInt64 num_Frames_without_ContentSize; + UInt64 windowSize_MAX; + UInt64 windowSize_Allocate_MAX; + // UInt64 num_DictionaryIds; + // UInt64 num_Blocks_forType[4]; + // UInt64 num_BlockBytes_forType[4]; + // UInt64 num_SingleSegments; + // UInt64 singleSegment_ContentSize_MAX; +} CZstdDecInfo; + +#define ZstdDecInfo_CLEAR(p) { memset(p, 0, sizeof(*(p))); } + +#define ZstdDecInfo_GET_NUM_FRAMES(p) ((p)->num_DataFrames + (p)->num_SkipFrames) + + +typedef struct CZstdDecState +{ + enum_ZstdStatus status; // out + Byte disableHash; + // Byte mustBeFinished; + Byte outSize_Defined; + // Byte isAfterSizeMode; + // UInt64 inProcessed; + // SRes codeRes; + // Byte needWrite_IsStrong; + + const Byte *inBuf; + size_t inPos; // in/out + size_t inLim; + + const Byte *win; // out + size_t winPos; // out + size_t wrPos; // in/out + // size_t cycSize; // out : if (!outBuf_fromCaller) + size_t needWrite_Size; // out + + Byte *outBuf_fromCaller; + size_t outBufSize_fromCaller; + /* (outBufSize_fromCaller >= full_uncompressed_size_of_all_frames) is required + for success decoding. + If outBufSize_fromCaller < full_uncompressed_size_of_all_frames), + decoding can give error message, because we decode per block basis. + */ + + // size_t outStep; + UInt64 outSize; // total in all frames + UInt64 outProcessed; // out decoded in all frames (it can be >= outSize) + + CZstdDecInfo info; +} CZstdDecState; + +void ZstdDecState_Clear(CZstdDecState *p); + +/* +ZstdDec_Decode() +return: + SZ_OK - no error + SZ_ERROR_DATA - Data Error + SZ_ERROR_MEM - Memory allocation error + SZ_ERROR_UNSUPPORTED - Unsupported method or method properties + SZ_ERROR_CRC - XXH hash Error + // SZ_ERROR_ARCHIVE - Headers error (not used now) +*/ +SRes ZstdDec_Decode(CZstdDecHandle dec, CZstdDecState *p); + +/* +ZstdDec_ReadUnusedFromInBuf(): +returns: the number of bytes that were read from InBuf +(*afterDecoding_tempPos) must be set to zero before first call of ZstdDec_ReadUnusedFromInBuf() +*/ +size_t ZstdDec_ReadUnusedFromInBuf( + CZstdDecHandle dec, + size_t afterDecoding_tempPos, // in/out + void *data, size_t size); + +typedef struct +{ + SRes decode_SRes; // error code of data decoding + Byte is_NonFinishedFrame; // there is unfinished decoding for data frame or skip frame + Byte extraSize; +} CZstdDecResInfo; + +/* +#define ZstdDecResInfo_CLEAR(p) \ +{ (p)->decode_SRes = 0; \ + (p)->is_NonFinishedFrame; \ + (p)->extraSize = 0; \ +} +// memset(p, 0, sizeof(*p)); +*/ + +/* +additional error codes for CZstdDecResInfo::decode_SRes: + SZ_ERROR_NO_ARCHIVE - is not zstd stream (no frames) + SZ_ERROR_INPUT_EOF - need more data in input stream +*/ +void ZstdDec_GetResInfo(const CZstdDec *dec, + const CZstdDecState *p, + SRes res, // it's result from ZstdDec_Decode() + CZstdDecResInfo *info); + +EXTERN_C_END + +#endif diff --git a/C/var_clang_arm64.mak b/C/var_clang_arm64.mak index 4b35409..971101a 100644 --- a/C/var_clang_arm64.mak +++ b/C/var_clang_arm64.mak @@ -6,6 +6,7 @@ IS_ARM64=1 CROSS_COMPILE= MY_ARCH= USE_ASM=1 +ASM_FLAGS=-Wno-unused-macros CC=$(CROSS_COMPILE)clang CXX=$(CROSS_COMPILE)clang++ USE_CLANG=1 diff --git a/CPP/7zip/7zip.mak b/CPP/7zip/7zip.mak index 6f4a0a1..8bf7e73 100644 --- a/CPP/7zip/7zip.mak +++ b/CPP/7zip/7zip.mak @@ -124,7 +124,7 @@ $(WIM_OBJS): ../../Archive/Wim/$(*B).cpp !IFDEF ZIP_OBJS $(ZIP_OBJS): ../../Archive/Zip/$(*B).cpp - $(COMPL) + $(COMPL) $(ZIP_FLAGS) !ENDIF !IFDEF COMPRESS_OBJS @@ -149,7 +149,7 @@ $(AGENT_OBJS): ../../UI/Agent/$(*B).cpp !IFDEF CONSOLE_OBJS $(CONSOLE_OBJS): ../../UI/Console/$(*B).cpp - $(COMPL) + $(COMPL) $(CONSOLE_VARIANT_FLAGS) !ENDIF !IFDEF EXPLORER_OBJS @@ -191,7 +191,7 @@ $(C_OBJS): ../../../../C/$(*B).c {../../UI/Agent}.cpp{$O}.obj:: $(COMPLB) {../../UI/Console}.cpp{$O}.obj:: - $(COMPLB) + $(COMPLB) $(CONSOLE_VARIANT_FLAGS) {../../UI/Explorer}.cpp{$O}.obj:: $(COMPLB) {../../UI/FileManager}.cpp{$O}.obj:: @@ -226,7 +226,7 @@ $(C_OBJS): ../../../../C/$(*B).c {../../Archive/Wim}.cpp{$O}.obj:: $(COMPLB) {../../Archive/Zip}.cpp{$O}.obj:: - $(COMPLB) + $(COMPLB) $(ZIP_FLAGS) {../../Compress}.cpp{$O}.obj:: $(COMPLB_O2) diff --git a/CPP/7zip/7zip_gcc.mak b/CPP/7zip/7zip_gcc.mak index 8bf0594..f6a044f 100644 --- a/CPP/7zip/7zip_gcc.mak +++ b/CPP/7zip/7zip_gcc.mak @@ -30,16 +30,33 @@ endif # for object file # -Wa,-aln=test.s # -save-temps +FLAGS_BASE = -mbranch-protection=standard -march=armv8.5-a +FLAGS_BASE = -mbranch-protection=standard +FLAGS_BASE = +# FLAGS_BASE = -DZ7_NO_UNICODE + CFLAGS_BASE_LIST = -c + + +#DEBUG_BUILD=1 + +ifdef DEBUG_BUILD +CFLAGS_DEBUG = -g +else +CFLAGS_DEBUG = -DNDEBUG +ifneq ($(CC), $(CROSS_COMPILE)clang) +LFLAGS_STRIP = -s +endif +endif + # CFLAGS_BASE_LIST = -S CFLAGS_BASE = -O2 $(CFLAGS_BASE_LIST) $(CFLAGS_WARN_WALL) $(CFLAGS_WARN) \ - -DNDEBUG -D_REENTRANT -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE \ + $(CFLAGS_DEBUG) -D_REENTRANT -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE \ -fPIC FLAGS_FLTO = -ffunction-sections FLAGS_FLTO = -flto -FLAGS_FLTO = -# +FLAGS_FLTO = $(FLAGS_BASE) # -DZ7_AFFINITY_DISABLE @@ -68,7 +85,7 @@ endif endif endif -LDFLAGS_STATIC = -DNDEBUG $(LDFLAGS_STATIC_2) +LDFLAGS_STATIC = $(CFLAGS_DEBUG) $(LDFLAGS_STATIC_2) $(LDFLAGS_STATIC_3) ifndef O ifdef IS_MINGW @@ -95,6 +112,7 @@ endif else LDFLAGS = $(LDFLAGS_STATIC) +# -z force-bti # -s is not required for clang, do we need it for GCC ??? #-static -static-libgcc -static-libstdc++ @@ -127,7 +145,8 @@ endif LIB2_GUI = -lOle32 -lGdi32 -lComctl32 -lComdlg32 -lShell32 $(LIB_HTMLHELP) LIB2 = -loleaut32 -luuid -ladvapi32 -lUser32 $(LIB2_GUI) -CXXFLAGS_EXTRA = -DUNICODE -D_UNICODE +# v24.00: -DUNICODE and -D_UNICODE are defined in precompilation header files +# CXXFLAGS_EXTRA = -DUNICODE -D_UNICODE # -Wno-delete-non-virtual-dtor @@ -142,6 +161,7 @@ DEL_OBJ_EXE = -$(RM) $(PROGPATH) $(PROGPATH_STATIC) $(OBJS) # LOCAL_LIBS=-lpthread # LOCAL_LIBS_DLL=$(LOCAL_LIBS) -ldl +LIB2 = -lpthread LIB2 = -lpthread -ldl @@ -190,7 +210,7 @@ CXX_WARN_FLAGS = #-Wno-invalid-offsetof #-Wno-reorder -CXXFLAGS = $(MY_ARCH_2) $(LOCAL_FLAGS) $(CXXFLAGS_BASE2) $(CFLAGS_BASE) $(FLAGS_FLTO) $(CXXFLAGS_EXTRA) $(CC_SHARED) $(CXX_WARN_FLAGS) $(CXX_STD_FLAGS) -o $@ +CXXFLAGS = $(MY_ARCH_2) $(LOCAL_FLAGS) $(CXXFLAGS_BASE2) $(CFLAGS_BASE) $(FLAGS_FLTO) $(CXXFLAGS_EXTRA) $(CC_SHARED) $(CXX_WARN_FLAGS) $(CXX_STD_FLAGS) $(CXX_INCLUDE_FLAGS) -o $@ STATIC_TARGET= ifdef COMPL_STATIC @@ -207,11 +227,28 @@ $(O): # LDFLAGS3= -Wl,--gc-sections # -Wl,--print-gc-sections -ifneq ($(CC), $(CROSS_COMPILE)clang) -LFLAGS_STRIP = -s +ifndef IS_MINGW + +# LFLAGS_NOEXECSTACK= + +ifdef Z7_USE_OS_UNAME_FOR_NOEXECSTACK +Z7_OS := $(shell uname) +show_os: + echo $(Z7_OS) + +# ifeq ($(CXX), $(CROSS_COMPILE)g++) +ifeq ($(Z7_OS), Linux) +LFLAGS_NOEXECSTACK ?= -z noexecstack +endif + +else +LFLAGS_NOEXECSTACK ?= $(shell echo 'int main(){return 0;}' | $(CC) $(MY_ARCH_2) -z noexecstack -o /dev/null -x c - 2>/dev/null && echo -z noexecstack || echo) +endif + endif -LFLAGS_ALL = $(LFLAGS_STRIP) $(MY_ARCH_2) $(LDFLAGS) $(FLAGS_FLTO) $(LD_arch) $(OBJS) $(MY_LIBS) $(LIB2) + +LFLAGS_ALL = $(LFLAGS_STRIP) $(MY_ARCH_2) $(LDFLAGS) $(FLAGS_FLTO) $(LD_arch) $(LFLAGS_NOEXECSTACK) $(OBJS) $(MY_LIBS) $(LIB2) # -s : GCC : Remove all symbol table and relocation information from the executable. # -s : CLANG : unsupported @@ -304,6 +341,8 @@ $O/XzCrc64Init.o: ../../../Common/XzCrc64Init.cpp $(CXX) $(CXXFLAGS) $< $O/XzCrc64Reg.o: ../../../Common/XzCrc64Reg.cpp $(CXX) $(CXXFLAGS) $< +$O/Xxh64Reg.o: ../../../Common/Xxh64Reg.cpp + $(CXX) $(CXXFLAGS) $< @@ -490,6 +529,8 @@ $O/IhexHandler.o: ../../Archive/IhexHandler.cpp $(CXX) $(CXXFLAGS) $< $O/LpHandler.o: ../../Archive/LpHandler.cpp $(CXX) $(CXXFLAGS) $< +$O/LvmHandler.o: ../../Archive/LvmHandler.cpp + $(CXX) $(CXXFLAGS) $< $O/LzhHandler.o: ../../Archive/LzhHandler.cpp $(CXX) $(CXXFLAGS) $< $O/LzmaHandler.o: ../../Archive/LzmaHandler.cpp @@ -536,6 +577,8 @@ $O/XzHandler.o: ../../Archive/XzHandler.cpp $(CXX) $(CXXFLAGS) $< $O/ZHandler.o: ../../Archive/ZHandler.cpp $(CXX) $(CXXFLAGS) $< +$O/ZstdHandler.o: ../../Archive/ZstdHandler.cpp + $(CXX) $(CXXFLAGS) $< $O/7zCompressionMode.o: ../../Archive/7z/7zCompressionMode.cpp @@ -642,7 +685,7 @@ $O/WimRegister.o: ../../Archive/Wim/WimRegister.cpp $O/ZipAddCommon.o: ../../Archive/Zip/ZipAddCommon.cpp $(CXX) $(CXXFLAGS) $< $O/ZipHandler.o: ../../Archive/Zip/ZipHandler.cpp - $(CXX) $(CXXFLAGS) $< + $(CXX) $(CXXFLAGS) $(ZIP_FLAGS) $< $O/ZipHandlerOut.o: ../../Archive/Zip/ZipHandlerOut.cpp $(CXX) $(CXXFLAGS) $< $O/ZipIn.o: ../../Archive/Zip/ZipIn.cpp @@ -763,6 +806,10 @@ $O/ZlibDecoder.o: ../../Compress/ZlibDecoder.cpp $(CXX) $(CXXFLAGS) $< $O/ZlibEncoder.o: ../../Compress/ZlibEncoder.cpp $(CXX) $(CXXFLAGS) $< +$O/ZstdDecoder.o: ../../Compress/ZstdDecoder.cpp + $(CXX) $(CXXFLAGS) $< +$O/ZstdRegister.o: ../../Compress/ZstdRegister.cpp + $(CXX) $(CXXFLAGS) $< $O/7zAes.o: ../../Crypto/7zAes.cpp @@ -953,6 +1000,8 @@ $O/App.o: ../../UI/FileManager/App.cpp $(CXX) $(CXXFLAGS) $< $O/BrowseDialog.o: ../../UI/FileManager/BrowseDialog.cpp $(CXX) $(CXXFLAGS) $< +$O/BrowseDialog2.o: ../../UI/FileManager/BrowseDialog2.cpp + $(CXX) $(CXXFLAGS) $< $O/ClassDefs.o: ../../UI/FileManager/ClassDefs.cpp $(CXX) $(CXXFLAGS) $< $O/ComboDialog.o: ../../UI/FileManager/ComboDialog.cpp @@ -993,6 +1042,8 @@ $O/LinkDialog.o: ../../UI/FileManager/LinkDialog.cpp $(CXX) $(CXXFLAGS) $< $O/ListViewDialog.o: ../../UI/FileManager/ListViewDialog.cpp $(CXX) $(CXXFLAGS) $< +$O/MemDialog.o: ../../UI/FileManager/MemDialog.cpp + $(CXX) $(CXXFLAGS) $< $O/MenuPage.o: ../../UI/FileManager/MenuPage.cpp $(CXX) $(CXXFLAGS) $< $O/MessagesDialog.o: ../../UI/FileManager/MessagesDialog.cpp @@ -1179,6 +1230,8 @@ $O/Sort.o: ../../../../C/Sort.c $(CC) $(CFLAGS) $< $O/SwapBytes.o: ../../../../C/SwapBytes.c $(CC) $(CFLAGS) $< +$O/Xxh64.o: ../../../../C/Xxh64.c + $(CC) $(CFLAGS) $< $O/Xz.o: ../../../../C/Xz.c $(CC) $(CFLAGS) $< $O/XzCrc64.o: ../../../../C/XzCrc64.c @@ -1189,6 +1242,8 @@ $O/XzEnc.o: ../../../../C/XzEnc.c $(CC) $(CFLAGS) $< $O/XzIn.o: ../../../../C/XzIn.c $(CC) $(CFLAGS) $< +$O/ZstdDec.o: ../../../../C/ZstdDec.c + $(CC) $(CFLAGS) $< ifdef USE_ASM @@ -1254,7 +1309,7 @@ endif ifdef IS_ARM64 $O/LzmaDecOpt.o: ../../../../Asm/arm64/LzmaDecOpt.S ../../../../Asm/arm64/7zAsm.S - $(CC) $(CFLAGS) $< + $(CC) $(CFLAGS) $(ASM_FLAGS) $< endif $O/LzmaDec.o: ../../../../C/LzmaDec.c diff --git a/CPP/7zip/Archive/7z/7zCompressionMode.h b/CPP/7zip/Archive/7z/7zCompressionMode.h index ac1f6ad..737722d 100644 --- a/CPP/7zip/Archive/7z/7zCompressionMode.h +++ b/CPP/7zip/Archive/7z/7zCompressionMode.h @@ -52,31 +52,30 @@ struct CCompressionMethodMode bool DefaultMethod_was_Inserted; bool Filter_was_Inserted; + bool PasswordIsDefined; + bool MemoryUsageLimit_WasSet; #ifndef Z7_ST - UInt32 NumThreads; bool NumThreads_WasForced; bool MultiThreadMixer; + UInt32 NumThreads; #endif - UInt64 MemoryUsageLimit; - bool MemoryUsageLimit_WasSet; - - bool PasswordIsDefined; UString Password; // _Wipe - + UInt64 MemoryUsageLimit; + bool IsEmpty() const { return (Methods.IsEmpty() && !PasswordIsDefined); } CCompressionMethodMode(): DefaultMethod_was_Inserted(false) , Filter_was_Inserted(false) + , PasswordIsDefined(false) + , MemoryUsageLimit_WasSet(false) #ifndef Z7_ST - , NumThreads(1) , NumThreads_WasForced(false) , MultiThreadMixer(true) + , NumThreads(1) #endif , MemoryUsageLimit((UInt64)1 << 30) - , MemoryUsageLimit_WasSet(false) - , PasswordIsDefined(false) {} #ifdef Z7_CPP_IS_SUPPORTED_default diff --git a/CPP/7zip/Archive/7z/7zDecode.cpp b/CPP/7zip/Archive/7z/7zDecode.cpp index 5420dce..50cbff6 100644 --- a/CPP/7zip/Archive/7z/7zDecode.cpp +++ b/CPP/7zip/Archive/7z/7zDecode.cpp @@ -442,9 +442,10 @@ HRESULT CDecoder::Decode( len = password.Len(); } CByteBuffer_Wipe buffer(len * 2); + const LPCOLESTR psw = passwordBSTR; for (size_t k = 0; k < len; k++) { - const wchar_t c = passwordBSTR[k]; + const wchar_t c = psw[k]; ((Byte *)buffer)[k * 2] = (Byte)c; ((Byte *)buffer)[k * 2 + 1] = (Byte)(c >> 8); } @@ -501,8 +502,7 @@ HRESULT CDecoder::Decode( CObjectVector< CMyComPtr > inStreams; - CLockedInStream *lockedInStreamSpec = new CLockedInStream; - CMyComPtr lockedInStream = lockedInStreamSpec; + CMyComPtr2_Create lockedInStream; #ifdef USE_MIXER_MT #ifdef USE_MIXER_ST @@ -514,8 +514,8 @@ HRESULT CDecoder::Decode( { // lockedInStream.Pos = (UInt64)(Int64)-1; // RINOK(InStream_GetPos(inStream, lockedInStream.Pos)) - RINOK(inStream->Seek((Int64)(startPos + packPositions[0]), STREAM_SEEK_SET, &lockedInStreamSpec->Pos)) - lockedInStreamSpec->Stream = inStream; + RINOK(inStream->Seek((Int64)(startPos + packPositions[0]), STREAM_SEEK_SET, &lockedInStream->Pos)) + lockedInStream->Stream = inStream; #ifdef USE_MIXER_MT #ifdef USE_MIXER_ST @@ -551,7 +551,7 @@ HRESULT CDecoder::Decode( { CLockedSequentialInStreamMT *lockedStreamImpSpec = new CLockedSequentialInStreamMT; packStream = lockedStreamImpSpec; - lockedStreamImpSpec->Init(lockedInStreamSpec, packPos); + lockedStreamImpSpec->Init(lockedInStream.ClsPtr(), packPos); } #ifdef USE_MIXER_ST else @@ -561,7 +561,7 @@ HRESULT CDecoder::Decode( #ifdef USE_MIXER_ST CLockedSequentialInStreamST *lockedStreamImpSpec = new CLockedSequentialInStreamST; packStream = lockedStreamImpSpec; - lockedStreamImpSpec->Init(lockedInStreamSpec, packPos); + lockedStreamImpSpec->Init(lockedInStream.ClsPtr(), packPos); #endif } } diff --git a/CPP/7zip/Archive/7z/7zEncode.cpp b/CPP/7zip/Archive/7z/7zEncode.cpp index 78c91cf..71d1ddb 100644 --- a/CPP/7zip/Archive/7z/7zEncode.cpp +++ b/CPP/7zip/Archive/7z/7zEncode.cpp @@ -299,10 +299,9 @@ static HRESULT FillProps_from_Coder(IUnknown *coder, CByteBuffer &props) writeCoderProperties, coder) if (writeCoderProperties) { - CDynBufSeqOutStream *outStreamSpec = new CDynBufSeqOutStream; - CMyComPtr dynOutStream(outStreamSpec); + CMyComPtr2_Create outStreamSpec; outStreamSpec->Init(); - RINOK(writeCoderProperties->WriteCoderProperties(dynOutStream)) + RINOK(writeCoderProperties->WriteCoderProperties(outStreamSpec)) outStreamSpec->CopyToBuffer(props); } else @@ -332,11 +331,8 @@ HRESULT CEncoder::Encode1( RINOK(_mixer->ReInit2()) - CMtEncMultiProgress *mtProgressSpec = NULL; - CMyComPtr mtProgress; - - CSequentialOutMtNotify *mtOutStreamNotifySpec = NULL; - CMyComPtr mtOutStreamNotify; + CMyComPtr2 mtProgress; + CMyComPtr2 mtOutStreamNotify; CRecordVector tempBufferSpecs; CObjectVector > tempBuffers; @@ -454,18 +450,16 @@ HRESULT CEncoder::Encode1( if (useMtProgress) { - mtProgressSpec = new CMtEncMultiProgress; - mtProgress = mtProgressSpec; - mtProgressSpec->Init(compressProgress); + mtProgress.SetFromCls(new CMtEncMultiProgress); + mtProgress->Init(compressProgress); - mtOutStreamNotifySpec = new CSequentialOutMtNotify; - mtOutStreamNotify = mtOutStreamNotifySpec; - mtOutStreamNotifySpec->_stream = outStream; - mtOutStreamNotifySpec->_mtProgressSpec = mtProgressSpec; + mtOutStreamNotify.SetFromCls(new CSequentialOutMtNotify); + mtOutStreamNotify->_stream = outStream; + mtOutStreamNotify->_mtProgressSpec = mtProgress.ClsPtr(); FOR_VECTOR (t, tempBufferSpecs) { - tempBufferSpecs[t]->_mtProgressSpec = mtProgressSpec; + tempBufferSpecs[t]->_mtProgressSpec = mtProgress.ClsPtr(); } } @@ -474,7 +468,8 @@ HRESULT CEncoder::Encode1( { outStreamSizeCountSpec = new CSequentialOutStreamSizeCount; outStreamSizeCount = outStreamSizeCountSpec; - outStreamSizeCountSpec->SetStream(mtOutStreamNotify ? (ISequentialOutStream *)mtOutStreamNotify : outStream); + outStreamSizeCountSpec->SetStream(mtOutStreamNotify.IsDefined() ? + mtOutStreamNotify.Interface() : outStream); outStreamSizeCountSpec->Init(); outStreamPointers.Add(outStreamSizeCount); } @@ -486,8 +481,9 @@ HRESULT CEncoder::Encode1( RINOK(_mixer->Code( &inStreamPointer, - &outStreamPointers.Front(), - mtProgress ? (ICompressProgressInfo *)mtProgress : compressProgress, dataAfterEnd_Error)) + outStreamPointers.ConstData(), + mtProgress.IsDefined() ? mtProgress.Interface() : + compressProgress, dataAfterEnd_Error)) if (_bindInfo.PackStreams.Size() != 0) packSizes.Add(outStreamSizeCountSpec->GetSize()); diff --git a/CPP/7zip/Archive/7z/7zExtract.cpp b/CPP/7zip/Archive/7z/7zExtract.cpp index 5498c59..1c31e1d 100644 --- a/CPP/7zip/Archive/7z/7zExtract.cpp +++ b/CPP/7zip/Archive/7z/7zExtract.cpp @@ -271,8 +271,7 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, RINOK(extractCallback->SetTotal(importantTotalUnpacked)) - CLocalProgress *lps = new CLocalProgress; - CMyComPtr progress = lps; + CMyComPtr2_Create lps; lps->Init(extractCallback, false); CDecoder decoder( @@ -385,7 +384,7 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, &curUnpacked, outStream, - progress, + lps, NULL // *inStreamMainRes , dataAfterEnd_Error diff --git a/CPP/7zip/Archive/7z/7zHandler.cpp b/CPP/7zip/Archive/7z/7zHandler.cpp index 23d3a9d..81dd966 100644 --- a/CPP/7zip/Archive/7z/7zHandler.cpp +++ b/CPP/7zip/Archive/7z/7zHandler.cpp @@ -272,6 +272,7 @@ Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) prop = true; break; } + default: break; } return prop.Detach(value); #ifndef Z7_SFX @@ -291,7 +292,7 @@ bool CHandler::IsFolderEncrypted(CNum folderIndex) const if (folderIndex == kNumNoIndex) return false; const size_t startPos = _db.FoCodersDataOffset[folderIndex]; - const Byte *p = _db.CodersData + startPos; + const Byte *p = _db.CodersData.ConstData() + startPos; const size_t size = _db.FoCodersDataOffset[folderIndex + 1] - startPos; CInByte2 inByte; inByte.Init(p, size); @@ -350,11 +351,11 @@ Z7_COM7F_IMF(CHandler::GetRawProp(UInt32 index, PROPID propID, const void **data { if (_db.NameOffsets && _db.NamesBuf) { - size_t offset = _db.NameOffsets[index]; - size_t size = (_db.NameOffsets[index + 1] - offset) * 2; + const size_t offset = _db.NameOffsets[index]; + const size_t size = (_db.NameOffsets[index + 1] - offset) * 2; if (size < ((UInt32)1 << 31)) { - *data = (const void *)(_db.NamesBuf + offset * 2); + *data = (const void *)(_db.NamesBuf.ConstData() + offset * 2); *dataSize = (UInt32)size; *propType = NPropDataType::kUtf16z; } @@ -395,7 +396,7 @@ HRESULT CHandler::SetMethodToProp(CNum folderIndex, PROPVARIANT *prop) const temp[--pos] = 0; const size_t startPos = _db.FoCodersDataOffset[folderIndex]; - const Byte *p = _db.CodersData + startPos; + const Byte *p = _db.CodersData.ConstData() + startPos; const size_t size = _db.FoCodersDataOffset[folderIndex + 1] - startPos; CInByte2 inByte; inByte.Init(p, size); @@ -482,9 +483,9 @@ HRESULT CHandler::SetMethodToProp(CNum folderIndex, PROPVARIANT *prop) const if (propsSize == 1) ConvertUInt32ToString((UInt32)props[0] + 1, s); } - else if (id == k_ARM64) + else if (id == k_ARM64 || id == k_RISCV) { - name = "ARM64"; + name = id == k_ARM64 ? "ARM64" : "RISCV"; if (propsSize == 4) ConvertUInt32ToString(GetUi32(props), s); /* @@ -666,6 +667,7 @@ Z7_COM7F_IMF(CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val #endif #endif + default: break; } // return prop.Detach(value); return S_OK; diff --git a/CPP/7zip/Archive/7z/7zHandler.h b/CPP/7zip/Archive/7z/7zHandler.h index b1c0466..ed535f7 100644 --- a/CPP/7zip/Archive/7z/7zHandler.h +++ b/CPP/7zip/Archive/7z/7zHandler.h @@ -54,11 +54,13 @@ class COutHandler: public CMultiMethodProps CBoolPair Write_Attrib; bool _useMultiThreadMixer; - bool _removeSfxBlock; - // bool _volumeMode; + UInt32 _decoderCompatibilityVersion; + CUIntVector _enabledFilters; + CUIntVector _disabledFilters; + void InitSolidFiles() { _numSolidFiles = (UInt64)(Int64)(-1); } void InitSolidSize() { _numSolidBytes = (UInt64)(Int64)(-1); } void InitSolid() diff --git a/CPP/7zip/Archive/7z/7zHandlerOut.cpp b/CPP/7zip/Archive/7z/7zHandlerOut.cpp index ff8735f..ea5ea0f 100644 --- a/CPP/7zip/Archive/7z/7zHandlerOut.cpp +++ b/CPP/7zip/Archive/7z/7zHandlerOut.cpp @@ -20,6 +20,9 @@ using namespace NWindows; namespace NArchive { namespace N7z { +static const UInt32 k_decoderCompatibilityVersion = 2301; +// 7-Zip version 2301 supports ARM64 filter + #define k_LZMA_Name "LZMA" #define kDefaultMethodName "LZMA2" #define k_Copy_Name "Copy" @@ -774,6 +777,11 @@ Z7_COM7F_IMF(CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt options.MaxFilter = (level >= 8); options.AnalysisLevel = GetAnalysisLevel(); + options.SetFilterSupporting_ver_enabled_disabled( + _decoderCompatibilityVersion, + _enabledFilters, + _disabledFilters); + options.HeaderOptions.CompressMainHeader = compressMainHeader; /* options.HeaderOptions.WriteCTime = Write_CTime; @@ -858,6 +866,10 @@ void COutHandler::InitProps7z() InitSolid(); _useTypeSorting = false; + + _decoderCompatibilityVersion = k_decoderCompatibilityVersion; + _enabledFilters.Clear(); + _disabledFilters.Clear(); } void COutHandler::InitProps() @@ -944,6 +956,29 @@ static HRESULT PROPVARIANT_to_BoolPair(const PROPVARIANT &prop, CBoolPair &dest) return S_OK; } +struct C_Id_Name_pair +{ + UInt32 Id; + const char *Name; +}; + +static const C_Id_Name_pair g_filter_pairs[] = +{ + { k_Delta, "Delta" }, + { k_ARM64, "ARM64" }, + { k_RISCV, "RISCV" }, + { k_SWAP2, "SWAP2" }, + { k_SWAP4, "SWAP4" }, + { k_BCJ, "BCJ" }, + { k_BCJ2 , "BCJ2" }, + { k_PPC, "PPC" }, + { k_IA64, "IA64" }, + { k_ARM, "ARM" }, + { k_ARMT, "ARMT" }, + { k_SPARC, "SPARC" } +}; + + HRESULT COutHandler::SetProperty(const wchar_t *nameSpec, const PROPVARIANT &value) { UString name = nameSpec; @@ -1004,6 +1039,43 @@ HRESULT COutHandler::SetProperty(const wchar_t *nameSpec, const PROPVARIANT &val if (name.IsEqualTo("qs")) return PROPVARIANT_to_bool(value, _useTypeSorting); + if (name.IsPrefixedBy_Ascii_NoCase("yv")) + { + name.Delete(0, 2); + UInt32 v = 1 << 16; // if no number is noit specified, we use big value + RINOK(ParsePropToUInt32(name, value, v)) + _decoderCompatibilityVersion = v; + // if (v == 0) _decoderCompatibilityVersion = k_decoderCompatibilityVersion; + return S_OK; + } + + if (name.IsPrefixedBy_Ascii_NoCase("yf")) + { + name.Delete(0, 2); + CUIntVector *vec; + if (name.IsEqualTo_Ascii_NoCase("a")) vec = &_enabledFilters; + else if (name.IsEqualTo_Ascii_NoCase("d")) vec = &_disabledFilters; + else return E_INVALIDARG; + + if (value.vt != VT_BSTR) + return E_INVALIDARG; + for (unsigned k = 0;; k++) + { + if (k == Z7_ARRAY_SIZE(g_filter_pairs)) + { + // maybe we can ignore unsupported filter names here? + return E_INVALIDARG; + } + const C_Id_Name_pair &pair = g_filter_pairs[k]; + if (StringsAreEqualNoCase_Ascii(value.bstrVal, pair.Name)) + { + vec->AddToUniqueSorted(pair.Id); + break; + } + } + return S_OK; + } + // if (name.IsEqualTo("v")) return PROPVARIANT_to_bool(value, _volumeMode); } return CMultiMethodProps::SetProperty(name, value); diff --git a/CPP/7zip/Archive/7z/7zHeader.h b/CPP/7zip/Archive/7z/7zHeader.h index bd96ca3..22e7960 100644 --- a/CPP/7zip/Archive/7z/7zHeader.h +++ b/CPP/7zip/Archive/7z/7zHeader.h @@ -100,6 +100,7 @@ namespace NID const UInt32 k_Copy = 0; const UInt32 k_Delta = 3; const UInt32 k_ARM64 = 0xa; +const UInt32 k_RISCV = 0xb; const UInt32 k_LZMA2 = 0x21; @@ -126,7 +127,7 @@ const UInt32 k_AES = 0x6F10701; // const UInt32 k_ZSTD = 0x4015D; // winzip zstd // 0x4F71101, 7z-zstd -static inline bool IsFilterMethod(UInt64 m) +inline bool IsFilterMethod(UInt64 m) { if (m > (UInt32)0xFFFFFFFF) return false; @@ -134,6 +135,7 @@ static inline bool IsFilterMethod(UInt64 m) { case k_Delta: case k_ARM64: + case k_RISCV: case k_BCJ: case k_BCJ2: case k_PPC: @@ -144,6 +146,7 @@ static inline bool IsFilterMethod(UInt64 m) case k_SWAP2: case k_SWAP4: return true; + default: break; } return false; } diff --git a/CPP/7zip/Archive/7z/7zIn.cpp b/CPP/7zip/Archive/7z/7zIn.cpp index 4defd27..e2efc52 100644 --- a/CPP/7zip/Archive/7z/7zIn.cpp +++ b/CPP/7zip/Archive/7z/7zIn.cpp @@ -32,37 +32,45 @@ using namespace NWindows; using namespace NCOM; -namespace NArchive { -namespace N7z { - -#define k_Scan_NumCoders_MAX 64 -#define k_Scan_NumCodersStreams_in_Folder_MAX 64 - unsigned BoolVector_CountSum(const CBoolVector &v); +Z7_NO_INLINE unsigned BoolVector_CountSum(const CBoolVector &v) { unsigned sum = 0; const unsigned size = v.Size(); - for (unsigned i = 0; i < size; i++) - if (v[i]) - sum++; + if (size) + { + const bool *p = v.ConstData(); + const bool * const lim = p + size; + do + if (*p) + sum++; + while (++p != lim); + } return sum; } static inline bool BoolVector_Item_IsValidAndTrue(const CBoolVector &v, unsigned i) { - return (i < v.Size() ? v[i] : false); + return i < v.Size() ? v[i] : false; } +Z7_NO_INLINE static void BoolVector_Fill_False(CBoolVector &v, unsigned size) { v.ClearAndSetSize(size); - bool *p = &v[0]; + bool *p = v.NonConstData(); for (unsigned i = 0; i < size; i++) p[i] = false; } +namespace NArchive { +namespace N7z { + +#define k_Scan_NumCoders_MAX 64 +#define k_Scan_NumCodersStreams_in_Folder_MAX 64 + class CInArchiveException {}; class CUnsupportedFeatureException: public CInArchiveException {}; @@ -510,7 +518,7 @@ void CFolders::ParseFolderInfo(unsigned folderIndex, CFolder &folder) const { const size_t startPos = FoCodersDataOffset[folderIndex]; CInByte2 inByte; - inByte.Init(CodersData + startPos, FoCodersDataOffset[folderIndex + 1] - startPos); + inByte.Init(CodersData.ConstData() + startPos, FoCodersDataOffset[folderIndex + 1] - startPos); inByte.ParseFolder(folder); if (inByte.GetRem() != 0) throw 20120424; @@ -1177,8 +1185,7 @@ HRESULT CInArchive::ReadAndDecodePackedStreams( ThrowUnsupported(); data.Alloc(unpackSize); - CBufPtrSeqOutStream *outStreamSpec = new CBufPtrSeqOutStream; - CMyComPtr outStream = outStreamSpec; + CMyComPtr2_Create outStreamSpec; outStreamSpec->Init(data, unpackSize); bool dataAfterEnd_Error = false; @@ -1189,7 +1196,7 @@ HRESULT CInArchive::ReadAndDecodePackedStreams( folders, i, NULL, // &unpackSize64 - outStream, + outStreamSpec, NULL, // *compressProgress NULL // **inStreamMainRes @@ -1280,7 +1287,7 @@ HRESULT CInArchive::ReadHeader( CBoolVector emptyStreamVector; CBoolVector emptyFileVector; CBoolVector antiFileVector; - CNum numEmptyStreams = 0; + unsigned numEmptyStreams = 0; for (;;) { @@ -1311,7 +1318,7 @@ HRESULT CInArchive::ReadHeader( for (i = 0; i < numFiles; i++) { const size_t curRem = (rem - pos) / 2; - const UInt16 *buf = (const UInt16 *)(const void *)(db.NamesBuf + pos); + const UInt16 *buf = (const UInt16 *)(const void *)(db.NamesBuf.ConstData() + pos); size_t j; for (j = 0; j < curRem && buf[j] != 0; j++); if (j == curRem) @@ -1457,7 +1464,7 @@ HRESULT CInArchive::ReadHeader( CNum emptyFileIndex = 0; CNum sizeIndex = 0; - const CNum numAntiItems = BoolVector_CountSum(antiFileVector); + const unsigned numAntiItems = BoolVector_CountSum(antiFileVector); if (numAntiItems != 0) db.IsAnti.ClearAndSetSize(numFiles); @@ -1649,7 +1656,7 @@ HRESULT CInArchive::ReadDatabase2( if (nextHeaderSize == 0) { - if (nextHeaderOffset != 0) + if (nextHeaderOffset != 0 || nextHeaderCRC != 0) return S_FALSE; db.IsArc = true; db.HeadersSize = HeadersSize; diff --git a/CPP/7zip/Archive/7z/7zIn.h b/CPP/7zip/Archive/7z/7zIn.h index a9c14fb..ddbe020 100644 --- a/CPP/7zip/Archive/7z/7zIn.h +++ b/CPP/7zip/Archive/7z/7zIn.h @@ -273,33 +273,36 @@ struct CDbEx: public CDatabase void FillLinks(); - UInt64 GetFolderStreamPos(CNum folderIndex, unsigned indexInFolder) const + UInt64 GetFolderStreamPos(size_t folderIndex, size_t indexInFolder) const { - return ArcInfo.DataStartPosition + - PackPositions[FoStartPackStreamIndex[folderIndex] + indexInFolder]; + return ArcInfo.DataStartPosition + PackPositions.ConstData() + [FoStartPackStreamIndex.ConstData()[folderIndex] + indexInFolder]; } - UInt64 GetFolderFullPackSize(CNum folderIndex) const + UInt64 GetFolderFullPackSize(size_t folderIndex) const { return - PackPositions[FoStartPackStreamIndex[folderIndex + 1]] - - PackPositions[FoStartPackStreamIndex[folderIndex]]; + PackPositions[FoStartPackStreamIndex.ConstData()[folderIndex + 1]] - + PackPositions[FoStartPackStreamIndex.ConstData()[folderIndex]]; } - UInt64 GetFolderPackStreamSize(CNum folderIndex, unsigned streamIndex) const + UInt64 GetFolderPackStreamSize(size_t folderIndex, size_t streamIndex) const { - size_t i = FoStartPackStreamIndex[folderIndex] + streamIndex; - return PackPositions[i + 1] - PackPositions[i]; + const size_t i = FoStartPackStreamIndex.ConstData()[folderIndex] + streamIndex; + return PackPositions.ConstData()[i + 1] - + PackPositions.ConstData()[i]; } - UInt64 GetFilePackSize(CNum fileIndex) const + /* + UInt64 GetFilePackSize(size_t fileIndex) const { - CNum folderIndex = FileIndexToFolderIndexMap[fileIndex]; + const CNum folderIndex = FileIndexToFolderIndexMap[fileIndex]; if (folderIndex != kNumNoIndex) if (FolderStartFileIndex[folderIndex] == fileIndex) return GetFolderFullPackSize(folderIndex); return 0; } + */ }; const unsigned kNumBufLevelsMax = 4; diff --git a/CPP/7zip/Archive/7z/7zItem.h b/CPP/7zip/Archive/7z/7zItem.h index e8c68be..06737c8 100644 --- a/CPP/7zip/Archive/7z/7zItem.h +++ b/CPP/7zip/Archive/7z/7zItem.h @@ -129,7 +129,7 @@ struct CUInt32DefVector bool CheckSize(unsigned size) const { return Defs.Size() == size || Defs.Size() == 0; } void SetItem(unsigned index, bool defined, UInt32 value); - void if_NonEmpty_FillResedue_with_false(unsigned numItems) + void if_NonEmpty_FillResidue_with_false(unsigned numItems) { if (Defs.Size() != 0 && Defs.Size() < numItems) SetItem(numItems - 1, false, 0); diff --git a/CPP/7zip/Archive/7z/7zOut.cpp b/CPP/7zip/Archive/7z/7zOut.cpp index 7f8fa5b..d0c8cf2 100644 --- a/CPP/7zip/Archive/7z/7zOut.cpp +++ b/CPP/7zip/Archive/7z/7zOut.cpp @@ -3,14 +3,34 @@ #include "StdAfx.h" #include "../../../../C/7zCrc.h" +#include "../../../../C/CpuArch.h" #include "../../../Common/AutoPtr.h" // #include "../../../Common/UTFConvert.h" #include "../../Common/StreamObjects.h" +#include "../Common/OutStreamWithCRC.h" #include "7zOut.h" +unsigned BoolVector_CountSum(const CBoolVector &v); + +static UInt64 UInt64Vector_CountSum(const CRecordVector &v) +{ + UInt64 sum = 0; + const unsigned size = v.Size(); + if (size) + { + const UInt64 *p = v.ConstData(); + const UInt64 * const lim = p + size; + do + sum += *p++; + while (p != lim); + } + return sum; +} + + namespace NArchive { namespace N7z { @@ -133,7 +153,7 @@ void COutArchive::WriteBytes(const void *data, size_t size) else if (_writeToStream) { _outByte.WriteBytes(data, size); - _crc = CrcUpdate(_crc, data, size); + // _crc = CrcUpdate(_crc, data, size); } else _outByte2.WriteBytes(data, size); @@ -144,14 +164,12 @@ void COutArchive::WriteByte(Byte b) if (_countMode) _countSize++; else if (_writeToStream) - { - _outByte.WriteByte(b); - _crc = CRC_UPDATE_BYTE(_crc, b); - } + WriteByte_ToStream(b); else _outByte2.WriteByte(b); } +/* void COutArchive::WriteUInt32(UInt32 value) { for (int i = 0; i < 4; i++) @@ -169,6 +187,7 @@ void COutArchive::WriteUInt64(UInt64 value) value >>= 8; } } +*/ void COutArchive::WriteNumber(UInt64 value) { @@ -288,7 +307,7 @@ void COutArchive::WriteFolder(const CFolder &folder) WriteNumber(folder.PackStreams[i]); } -void COutArchive::WriteBoolVector(const CBoolVector &boolVector) +void COutArchive::Write_BoolVector(const CBoolVector &boolVector) { Byte b = 0; Byte mask = 0x80; @@ -314,31 +333,32 @@ void COutArchive::WritePropBoolVector(Byte id, const CBoolVector &boolVector) { WriteByte(id); WriteNumber(Bv_GetSizeInBytes(boolVector)); - WriteBoolVector(boolVector); + Write_BoolVector(boolVector); +} + +void COutArchive::Write_BoolVector_numDefined(const CBoolVector &boolVector, unsigned numDefined) +{ + if (numDefined == boolVector.Size()) + WriteByte(1); + else + { + WriteByte(0); + Write_BoolVector(boolVector); + } } -unsigned BoolVector_CountSum(const CBoolVector &v); void COutArchive::WriteHashDigests(const CUInt32DefVector &digests) { const unsigned numDefined = BoolVector_CountSum(digests.Defs); if (numDefined == 0) return; - WriteByte(NID::kCRC); - if (numDefined == digests.Defs.Size()) - WriteByte(1); - else - { - WriteByte(0); - WriteBoolVector(digests.Defs); - } - - for (unsigned i = 0; i < digests.Defs.Size(); i++) - if (digests.Defs[i]) - WriteUInt32(digests.Vals[i]); + Write_BoolVector_numDefined(digests.Defs, numDefined); + Write_UInt32DefVector_numDefined(digests, numDefined); } + void COutArchive::WritePackInfo( UInt64 dataOffset, const CRecordVector &packSizes, @@ -467,17 +487,42 @@ void COutArchive::WriteAlignedBools(const CBoolVector &v, unsigned numDefined, B WriteByte(type); WriteNumber(dataSize); - if (numDefined == v.Size()) - WriteByte(1); - else + Write_BoolVector_numDefined(v, numDefined); + WriteByte(0); // 0 means no switching to external stream +} + + +void COutArchive::Write_UInt32DefVector_numDefined(const CUInt32DefVector &v, unsigned numDefined) +{ + if (_countMode) { - WriteByte(0); - WriteBoolVector(v); + _countSize += (size_t)numDefined * 4; + return; } - WriteByte(0); // 0 means no switching to external stream + + const bool * const defs = v.Defs.ConstData(); + const UInt32 * const vals = v.Vals.ConstData(); + const size_t num = v.Defs.Size(); + + for (size_t i = 0; i < num; i++) + if (defs[i]) + { + UInt32 value = vals[i]; + for (int k = 0; k < 4; k++) + { + if (_writeToStream) + WriteByte_ToStream((Byte)value); + else + _outByte2.WriteByte((Byte)value); + // WriteByte((Byte)value); + value >>= 8; + } + // WriteUInt32(v.Vals[i]); + } } -void COutArchive::WriteUInt64DefVector(const CUInt64DefVector &v, Byte type) + +void COutArchive::Write_UInt64DefVector_type(const CUInt64DefVector &v, Byte type) { const unsigned numDefined = BoolVector_CountSum(v.Defs); if (numDefined == 0) @@ -485,18 +530,40 @@ void COutArchive::WriteUInt64DefVector(const CUInt64DefVector &v, Byte type) WriteAlignedBools(v.Defs, numDefined, type, 3); - for (unsigned i = 0; i < v.Defs.Size(); i++) - if (v.Defs[i]) - WriteUInt64(v.Vals[i]); + if (_countMode) + { + _countSize += (size_t)numDefined * 8; + return; + } + + const bool * const defs = v.Defs.ConstData(); + const UInt64 * const vals = v.Vals.ConstData(); + const size_t num = v.Defs.Size(); + + for (size_t i = 0; i < num; i++) + if (defs[i]) + { + UInt64 value = vals[i]; + for (int k = 0; k < 8; k++) + { + if (_writeToStream) + WriteByte_ToStream((Byte)value); + else + _outByte2.WriteByte((Byte)value); + // WriteByte((Byte)value); + value >>= 8; + } + // WriteUInt64(v.Vals[i]); + } } + HRESULT COutArchive::EncodeStream( DECL_EXTERNAL_CODECS_LOC_VARS CEncoder &encoder, const CByteBuffer &data, CRecordVector &packSizes, CObjectVector &folders, COutFolders &outFolders) { - CBufInStream *streamSpec = new CBufInStream; - CMyComPtr stream = streamSpec; + CMyComPtr2_Create streamSpec; streamSpec->Init(data, data.Size()); outFolders.FolderUnpackCRCs.Defs.Add(true); outFolders.FolderUnpackCRCs.Vals.Add(CrcCalc(data, data.Size())); @@ -505,7 +572,7 @@ HRESULT COutArchive::EncodeStream( const UInt64 expectSize = data.Size(); RINOK(encoder.Encode1( EXTERNAL_CODECS_LOC_VARS - stream, + streamSpec, // NULL, &dataSize64, // inSizeForReduce expectSize, @@ -528,13 +595,7 @@ void COutArchive::WriteHeader( */ _useAlign = true; - { - UInt64 packSize = 0; - FOR_VECTOR (i, db.PackSizes) - packSize += db.PackSizes[i]; - headerOffset = packSize; - } - + headerOffset = UInt64Vector_CountSum(db.PackSizes); WriteByte(NID::kHeader); @@ -655,81 +716,97 @@ void COutArchive::WriteHeader( { /* ---------- Names ---------- */ - unsigned numDefined = 0; size_t namesDataSize = 0; - FOR_VECTOR (i, db.Files) { - const UString &name = db.Names[i]; - if (!name.IsEmpty()) - numDefined++; - const size_t numUtfChars = - /* - #if WCHAR_MAX > 0xffff + FOR_VECTOR (i, db.Files) + { + const UString &name = db.Names[i]; + const size_t numUtfChars = + /* + #if WCHAR_MAX > 0xffff Get_Num_Utf16_chars_from_wchar_string(name.Ptr()); - #else - */ + #else + */ name.Len(); - // #endif - namesDataSize += (numUtfChars + 1) * 2; + // #endif + namesDataSize += numUtfChars; + } } - - if (numDefined > 0) + if (namesDataSize) { - namesDataSize++; + namesDataSize += db.Files.Size(); // we will write tail zero wchar for each name + namesDataSize *= 2; // 2 bytes per wchar for UTF16 encoding + namesDataSize++; // for additional switch byte (zero value) SkipToAligned(2 + GetBigNumberSize(namesDataSize), 4); - WriteByte(NID::kName); WriteNumber(namesDataSize); - WriteByte(0); - FOR_VECTOR (i, db.Files) + + if (_countMode) + _countSize += namesDataSize; + else { - const UString &name = db.Names[i]; - for (unsigned t = 0; t <= name.Len(); t++) + WriteByte(0); + FOR_VECTOR (i, db.Files) { - wchar_t c = name[t]; - - /* - #if WCHAR_MAX > 0xffff - if (c >= 0x10000) + const UString &name = db.Names[i]; + const wchar_t *p = name.Ptr(); + const size_t len = (size_t)name.Len() + 1; + const wchar_t * const lim = p + len; + if (_writeToStream) { - c -= 0x10000; - if (c < (1 << 20)) + do { - unsigned c0 = 0xd800 + ((c >> 10) & 0x3FF); - WriteByte((Byte)c0); - WriteByte((Byte)(c0 >> 8)); - c = 0xdc00 + (c & 0x3FF); + const wchar_t c = *p++; + WriteByte_ToStream((Byte)c); + WriteByte_ToStream((Byte)(c >> 8)); } - else - c = '_'; // we change character unsupported by UTF16 + while (p != lim); + } + else + { + Byte *dest = _outByte2.GetDest_and_Update(len * 2); + do + { + /* + #if WCHAR_MAX > 0xffff + if (c >= 0x10000) + { + c -= 0x10000; + if (c < (1 << 20)) + { + unsigned c0 = 0xd800 + ((c >> 10) & 0x3FF); + WriteByte((Byte)c0); + WriteByte((Byte)(c0 >> 8)); + c = 0xdc00 + (c & 0x3FF); + } + else + c = '_'; // we change character unsupported by UTF16 + } + #endif + */ + const wchar_t c = *p++; + SetUi16(dest, (UInt16)c) + dest += 2; + } + while (p != lim); } - #endif - */ - - WriteByte((Byte)c); - WriteByte((Byte)(c >> 8)); } } } } - /* if (headerOptions.WriteCTime) */ WriteUInt64DefVector(db.CTime, NID::kCTime); - /* if (headerOptions.WriteATime) */ WriteUInt64DefVector(db.ATime, NID::kATime); - /* if (headerOptions.WriteMTime) */ WriteUInt64DefVector(db.MTime, NID::kMTime); - WriteUInt64DefVector(db.StartPos, NID::kStartPos); + /* if (headerOptions.WriteCTime) */ Write_UInt64DefVector_type(db.CTime, NID::kCTime); + /* if (headerOptions.WriteATime) */ Write_UInt64DefVector_type(db.ATime, NID::kATime); + /* if (headerOptions.WriteMTime) */ Write_UInt64DefVector_type(db.MTime, NID::kMTime); + Write_UInt64DefVector_type(db.StartPos, NID::kStartPos); { /* ---------- Write Attrib ---------- */ const unsigned numDefined = BoolVector_CountSum(db.Attrib.Defs); - if (numDefined != 0) { WriteAlignedBools(db.Attrib.Defs, numDefined, NID::kWinAttrib, 2); - FOR_VECTOR (i, db.Attrib.Defs) - { - if (db.Attrib.Defs[i]) - WriteUInt32(db.Attrib.Vals[i]); - } + Write_UInt32DefVector_numDefined(db.Attrib, numDefined); } } @@ -765,13 +842,7 @@ void COutArchive::WriteHeader( WriteByte(NID::kParent); WriteNumber(dataSize); - if (numIsDir == boolVector.Size()) - WriteByte(1); - else - { - WriteByte(0); - WriteBoolVector(boolVector); - } + Write_BoolVector_numDefined(boolVector, numIsDir); for (i = 0; i < db.Files.Size(); i++) { const CFileItem &file = db.Files[i]; @@ -818,17 +889,17 @@ HRESULT COutArchive::WriteDatabase( if (!db.CheckNumFiles()) return E_FAIL; - UInt64 headerOffset; - UInt32 headerCRC; - UInt64 headerSize; - if (db.IsEmpty()) - { - headerSize = 0; - headerOffset = 0; - headerCRC = CrcCalc(NULL, 0); - } - else + CStartHeader sh; + sh.NextHeaderOffset = 0; + sh.NextHeaderSize = 0; + sh.NextHeaderCRC = 0; // CrcCalc(NULL, 0); + + if (!db.IsEmpty()) { + CMyComPtr2_Create crcStream; + crcStream->SetStream(SeqStream); + crcStream->Init(); + bool encodeHeaders = false; if (options) if (options->IsEmpty()) @@ -837,13 +908,15 @@ HRESULT COutArchive::WriteDatabase( if (options->PasswordIsDefined || headerOptions.CompressMainHeader) encodeHeaders = true; - _outByte.SetStream(SeqStream); + if (!_outByte.Create(1 << 16)) + return E_OUTOFMEMORY; + _outByte.SetStream(crcStream.Interface()); _outByte.Init(); - _crc = CRC_INIT_VAL; + // _crc = CRC_INIT_VAL; _countMode = encodeHeaders; _writeToStream = true; _countSize = 0; - WriteHeader(db, /* headerOptions, */ headerOffset); + WriteHeader(db, /* headerOptions, */ sh.NextHeaderOffset); if (encodeHeaders) { @@ -852,7 +925,7 @@ HRESULT COutArchive::WriteDatabase( _countMode = false; _writeToStream = false; - WriteHeader(db, /* headerOptions, */ headerOffset); + WriteHeader(db, /* headerOptions, */ sh.NextHeaderOffset); if (_countSize != _outByte2.GetPos()) return E_FAIL; @@ -876,15 +949,17 @@ HRESULT COutArchive::WriteDatabase( throw 1; WriteID(NID::kEncodedHeader); - WritePackInfo(headerOffset, packSizes, CUInt32DefVector()); + WritePackInfo(sh.NextHeaderOffset, packSizes, CUInt32DefVector()); WriteUnpackInfo(folders, outFolders); WriteByte(NID::kEnd); - FOR_VECTOR (i, packSizes) - headerOffset += packSizes[i]; + + sh.NextHeaderOffset += UInt64Vector_CountSum(packSizes); } RINOK(_outByte.Flush()) - headerCRC = CRC_GET_DIGEST(_crc); - headerSize = _outByte.GetProcessedSize(); + sh.NextHeaderCRC = crcStream->GetCRC(); + // sh.NextHeaderCRC = CRC_GET_DIGEST(_crc); + // if (CRC_GET_DIGEST(_crc) != sh.NextHeaderCRC) throw 1; + sh.NextHeaderSize = _outByte.GetProcessedSize(); } #ifdef Z7_7Z_VOL if (_endMarker) @@ -904,12 +979,8 @@ HRESULT COutArchive::WriteDatabase( #endif if (Stream) { - CStartHeader h; - h.NextHeaderSize = headerSize; - h.NextHeaderCRC = headerCRC; - h.NextHeaderOffset = headerOffset; RINOK(Stream->Seek((Int64)_signatureHeaderPos, STREAM_SEEK_SET, NULL)) - return WriteStartHeader(h); + return WriteStartHeader(sh); } return S_OK; } diff --git a/CPP/7zip/Archive/7z/7zOut.h b/CPP/7zip/Archive/7z/7zOut.h index 940cafc..c65d268 100644 --- a/CPP/7zip/Archive/7z/7zOut.h +++ b/CPP/7zip/Archive/7z/7zOut.h @@ -19,32 +19,41 @@ const unsigned k_StartHeadersRewriteSize = 32; class CWriteBufferLoc { Byte *_data; - size_t _size; - size_t _pos; + Byte *_dataLim; + Byte *_dataBase; public: - CWriteBufferLoc(): _size(0), _pos(0) {} + // CWriteBufferLoc(): _data(NULL), _dataLim(NULL), _dataBase(NULL) {} void Init(Byte *data, size_t size) { _data = data; - _size = size; - _pos = 0; + _dataBase = data; + _dataLim = data + size; + } + + Byte *GetDest_and_Update(size_t size) + { + Byte *dest = _data; + if (size > (size_t)(_dataLim - dest)) + throw 1; + _data = dest + size; + return dest; } void WriteBytes(const void *data, size_t size) { if (size == 0) return; - if (size > _size - _pos) - throw 1; - memcpy(_data + _pos, data, size); - _pos += size; + Byte *dest = GetDest_and_Update(size); + memcpy(dest, data, size); } void WriteByte(Byte b) { - if (_size == _pos) + Byte *dest = _data; + if (dest == _dataLim) throw 1; - _data[_pos++] = b; + *dest++ = b; + _data = dest; } - size_t GetPos() const { return _pos; } + size_t GetPos() const { return (size_t)(_data - _dataBase); } }; @@ -248,14 +257,20 @@ class COutArchive void WriteBytes(const void *data, size_t size); void WriteBytes(const CByteBuffer &data) { WriteBytes(data, data.Size()); } void WriteByte(Byte b); - void WriteUInt32(UInt32 value); - void WriteUInt64(UInt64 value); + void WriteByte_ToStream(Byte b) + { + _outByte.WriteByte(b); + // _crc = CRC_UPDATE_BYTE(_crc, b); + } + // void WriteUInt32(UInt32 value); + // void WriteUInt64(UInt64 value); void WriteNumber(UInt64 value); void WriteID(UInt64 value) { WriteNumber(value); } void WriteFolder(const CFolder &folder); HRESULT WriteFileHeader(const CFileItem &itemInfo); - void WriteBoolVector(const CBoolVector &boolVector); + void Write_BoolVector(const CBoolVector &boolVector); + void Write_BoolVector_numDefined(const CBoolVector &boolVector, unsigned numDefined); void WritePropBoolVector(Byte id, const CBoolVector &boolVector); void WriteHashDigests(const CUInt32DefVector &digests); @@ -277,7 +292,8 @@ class COutArchive void SkipToAligned(unsigned pos, unsigned alignShifts); void WriteAlignedBools(const CBoolVector &v, unsigned numDefined, Byte type, unsigned itemSizeShifts); - void WriteUInt64DefVector(const CUInt64DefVector &v, Byte type); + void Write_UInt32DefVector_numDefined(const CUInt32DefVector &v, unsigned numDefined); + void Write_UInt64DefVector_type(const CUInt64DefVector &v, Byte type); HRESULT EncodeStream( DECL_EXTERNAL_CODECS_LOC_VARS @@ -294,7 +310,7 @@ class COutArchive #ifdef Z7_7Z_VOL bool _endMarker; #endif - UInt32 _crc; + // UInt32 _crc; size_t _countSize; CWriteBufferLoc _outByte2; COutBuffer _outByte; @@ -310,7 +326,7 @@ class COutArchive public: CMyComPtr SeqStream; - COutArchive() { _outByte.Create(1 << 16); } + // COutArchive(); HRESULT Create_and_WriteStartPrefix(ISequentialOutStream *stream /* , bool endMarker */); void Close(); HRESULT WriteDatabase( diff --git a/CPP/7zip/Archive/7z/7zUpdate.cpp b/CPP/7zip/Archive/7z/7zUpdate.cpp index 4e9ea5d..d374a00 100644 --- a/CPP/7zip/Archive/7z/7zUpdate.cpp +++ b/CPP/7zip/Archive/7z/7zUpdate.cpp @@ -33,7 +33,7 @@ struct CFilterMode UInt32 Id; UInt32 Delta; // required File Size alignment, if Id is not k_Delta. // (Delta == 0) means unknown alignment - UInt32 Offset; // for k_ARM64 + UInt32 Offset; // for k_ARM64 / k_RISCV // UInt32 AlignSizeOpt; // for k_ARM64 CFilterMode(): @@ -59,7 +59,7 @@ struct CFilterMode Delta = 16; else if (Id == k_ARM64 || Id == k_ARM || Id == k_PPC || Id == k_SPARC) Delta = 4; - else if (Id == k_ARMT) + else if (Id == k_ARMT || Id == k_RISCV) Delta = 2; else if (Id == k_BCJ || Id == k_BCJ2) Delta = 1; // do we need it? @@ -95,7 +95,9 @@ static int Parse_EXE(const Byte *buf, size_t size, CFilterMode *filterMode) return 0; p += 4; - switch (GetUi16(p)) + const unsigned machine = GetUi16(p); + + switch (machine) { case 0x014C: case 0x8664: filterId = k_X86; break; @@ -112,11 +114,16 @@ static int Parse_EXE(const Byte *buf, size_t size, CFilterMode *filterMode) case 0x01C2: filterId = k_ARM; break; // WinCE new case 0x01C4: filterId = k_ARMT; break; // WinRT + case 0x5032: // RISCV32 + case 0x5064: // RISCV64 + // case 0x5128: // RISCV128 + filterId = k_RISCV; break; + case 0x0200: filterId = k_IA64; break; default: return 0; } - // const UInt32 numSections = GetUi16(p + 2); + const UInt32 numSections = GetUi16(p + 2); optHeaderSize = GetUi16(p + 16); if (optHeaderSize > (1 << 10)) return 0; @@ -132,34 +139,49 @@ static int Parse_EXE(const Byte *buf, size_t size, CFilterMode *filterMode) return 0; } - /* // Windows exe file sizes are not aligned for 4 KiB. // So we can't use (CFilterMode::Offset != 0) in solid archives. // So we just don't set Offset here. #define NUM_SCAN_SECTIONS_MAX (1 << 6) -#define EXE_SECTION_OFFSET_MAX (1 << 27) -#define EXE_SECTION_SIZE_MIN (1 << 8) -#define EXE_SECTION_SIZE_MAX (1 << 27) +// #define EXE_SECTION_OFFSET_MAX (1 << 27) +// #define EXE_SECTION_SIZE_MIN (1 << 8) +// #define EXE_SECTION_SIZE_MAX (1 << 27) #define PE_SectHeaderSize 40 -#define PE_SECT_EXECUTE 0x20000000 +// #define PE_SECT_EXECUTE 0x20000000 +/* if (numSections > NUM_SCAN_SECTIONS_MAX) return 0; +*/ + if ((size_t)(p - buf) + optHeaderSize <= size) + { p += optHeaderSize; +/* // UInt32 numExeSections = 0; // bool execute_finded = false; // UInt32 sect_va = 0; // UInt32 sect_size = 0; // UInt32 sect_offset = 0; - +*/ + if (numSections <= NUM_SCAN_SECTIONS_MAX) + if (machine == 0x8664) for (UInt32 i = 0; i < numSections - // && numExeSections < numSectionsMax ; i++, p += PE_SectHeaderSize) { - UInt32 characts, rawSize, offset; + // UInt32 characts, rawSize, offset; if ((UInt32)(p - buf) + PE_SectHeaderSize > size) - return 0; + { + // return 0; + break; + } + if (memcmp(p, ".a64xrm", 8) == 0) + { + // ARM64EC + filterId = k_ARM64; + break; + } +/* rawSize = GetUi32(p + 16); offset = GetUi32(p + 20); characts = GetUi32(p + 36); @@ -178,8 +200,11 @@ static int Parse_EXE(const Byte *buf, size_t size, CFilterMode *filterMode) break; } } +*/ + } } + /* filterMode->Offset = 0; if (filterId == k_ARM64) { @@ -243,8 +268,9 @@ static int Parse_ELF(const Byte *buf, size_t size, CFilterMode *filterMode) case 43: filterId = k_SPARC; break; case 20: case 21: if (!be) return 0; filterId = k_PPC; break; - case 40: if ( be) return 0; filterId = k_ARM; break; + case 40: if (be) return 0; filterId = k_ARM; break; case 183: if (be) return 0; filterId = k_ARM64; break; + case 243: if (be) return 0; filterId = k_RISCV; break; /* Some IA-64 ELF executables have size that is not aligned for 16 bytes. So we don't use IA-64 filter for IA-64 ELF */ @@ -482,6 +508,7 @@ static inline bool IsExeFilter(CMethodId m) switch (m) { case k_ARM64: + case k_RISCV: case k_BCJ: case k_BCJ2: case k_ARM: @@ -490,6 +517,7 @@ static inline bool IsExeFilter(CMethodId m) case k_SPARC: case k_IA64: return true; + default: break; } return false; } @@ -521,7 +549,8 @@ static unsigned Get_FilterGroup_for_Folder( if (m.Id == k_BCJ2) m.Id = k_BCJ; m.SetDelta(); - if (m.Id == k_ARM64) + if (m.Id == k_ARM64 || + m.Id == k_RISCV) if (coder.Props.Size() == 4) m.Offset = GetUi32(coder.Props); } @@ -537,15 +566,12 @@ static HRESULT WriteRange(IInStream *inStream, ISequentialOutStream *outStream, UInt64 position, UInt64 size, ICompressProgressInfo *progress) { RINOK(InStream_SeekSet(inStream, position)) - CLimitedSequentialInStream *streamSpec = new CLimitedSequentialInStream; - CMyComPtr inStreamLimited(streamSpec); + CMyComPtr2_Create streamSpec; streamSpec->SetStream(inStream); streamSpec->Init(size); - - NCompress::CCopyCoder *copyCoderSpec = new NCompress::CCopyCoder; - CMyComPtr copyCoder = copyCoderSpec; - RINOK(copyCoder->Code(inStreamLimited, outStream, NULL, NULL, progress)) - return (copyCoderSpec->TotalSize == size ? S_OK : E_FAIL); + CMyComPtr2_Create copyCoder; + RINOK(copyCoder.Interface()->Code(streamSpec, outStream, NULL, NULL, progress)) + return (copyCoder->TotalSize == size ? S_OK : E_FAIL); } /* @@ -759,9 +785,9 @@ struct CRefItem { if (sortByType) { - int slashPos = ui.Name.ReverseFind_PathSepar(); + const int slashPos = ui.Name.ReverseFind_PathSepar(); NamePos = (unsigned)(slashPos + 1); - int dotPos = ui.Name.ReverseFind_Dot(); + const int dotPos = ui.Name.ReverseFind_Dot(); if (dotPos <= slashPos) ExtensionPos = ui.Name.Len(); else @@ -772,7 +798,7 @@ struct CRefItem AString s; for (unsigned pos = ExtensionPos;; pos++) { - wchar_t c = ui.Name[pos]; + const wchar_t c = ui.Name[pos]; if (c >= 0x80) break; if (c == 0) @@ -780,7 +806,7 @@ struct CRefItem ExtensionIndex = GetExtIndex(s); break; } - s += (char)MyCharLower_Ascii((char)c); + s.Add_Char((char)MyCharLower_Ascii((char)c)); } } } @@ -1123,6 +1149,12 @@ HRESULT CAnalysis::GetFilterGroup(UInt32 index, const CUpdateItem &ui, CFilterMo #ifdef MY_CPU_ARM64 filterModeTemp.Id = k_ARM64; #endif + #ifdef MY_CPU_RISCV + filterModeTemp.Id = k_RISCV; + #endif + #ifdef MY_CPU_SPARC + filterModeTemp.Id = k_SPARC; + #endif parseRes = true; } @@ -1172,6 +1204,8 @@ static inline void GetMethodFull(UInt64 methodID, UInt32 numStreams, CMethodFull m.NumStreams = numStreams; } + +// we add bond for mode.Methods[0] that is filter static HRESULT AddBondForFilter(CCompressionMethodMode &mode) { for (unsigned c = 1; c < mode.Methods.Size(); c++) @@ -1189,16 +1223,19 @@ static HRESULT AddBondForFilter(CCompressionMethodMode &mode) return E_INVALIDARG; } -static HRESULT AddFilterBond(CCompressionMethodMode &mode) +/* +static HRESULT AddBondForFilter_if_ThereAreBonds(CCompressionMethodMode &mode) { if (!mode.Bonds.IsEmpty()) return AddBondForFilter(mode); return S_OK; } +*/ static HRESULT AddBcj2Methods(CCompressionMethodMode &mode) { // mode.Methods[0] must be k_BCJ2 method ! + // mode.Methods[1] : we expect that there is at least one method after BCJ2 CMethodFull m; GetMethodFull(k_LZMA, 1, m); @@ -1210,7 +1247,7 @@ static HRESULT AddBcj2Methods(CCompressionMethodMode &mode) m.AddProp32(NCoderPropID::kLitContextBits, 0); // m.AddProp_Ascii(NCoderPropID::kMatchFinder, "BT2"); - unsigned methodIndex = mode.Methods.Size(); + const unsigned methodIndex = mode.Methods.Size(); if (mode.Bonds.IsEmpty()) { @@ -1229,109 +1266,140 @@ static HRESULT AddBcj2Methods(CCompressionMethodMode &mode) RINOK(AddBondForFilter(mode)) CBond2 bond; - bond.OutCoder = 0; + bond.OutCoder = 0; // index of BCJ2 coder bond.InCoder = methodIndex; bond.OutStream = 1; mode.Bonds.Add(bond); bond.InCoder = methodIndex + 1; bond.OutStream = 2; mode.Bonds.Add(bond); return S_OK; } + static HRESULT MakeExeMethod(CCompressionMethodMode &mode, - const CFilterMode &filterMode, /* bool addFilter, */ bool bcj2Filter) + const CFilterMode &filterMode, + const bool bcj2_IsAllowed, + const CUIntVector &disabledFilterIDs) { if (mode.Filter_was_Inserted) { + // filter was inserted, but bond for that filter was not added still. const CMethodFull &m = mode.Methods[0]; - const CMethodId id = m.Id; - if (id == k_BCJ2) + if (m.Id == k_BCJ2) return AddBcj2Methods(mode); if (!m.IsSimpleCoder()) return E_NOTIMPL; - // if (Bonds.IsEmpty()) we can create bonds later - return AddFilterBond(mode); + if (mode.Bonds.IsEmpty()) + return S_OK; + return AddBondForFilter(mode); } if (filterMode.Id == 0) return S_OK; - CMethodFull &m = mode.Methods.InsertNew(0); + unsigned nextCoder; - { - FOR_VECTOR (k, mode.Bonds) - { - CBond2 &bond = mode.Bonds[k]; - bond.InCoder++; - bond.OutCoder++; - } - } + const bool useBcj2 = bcj2_IsAllowed + && Is86Filter(filterMode.Id) + && disabledFilterIDs.FindInSorted(k_BCJ2) < 0; - HRESULT res; - - if (bcj2Filter && Is86Filter(filterMode.Id)) + if (!useBcj2 && disabledFilterIDs.FindInSorted(filterMode.Id) >= 0) { - GetMethodFull(k_BCJ2, 4, m); - res = AddBcj2Methods(mode); + // required filter is disabled, + // but we still can use information about data alignment. +#if 0 // 1 for debug + // we can return here, if we want default lzma properties + return S_OK; +#else + // we will try to change lzma/lzma2 properties + nextCoder = 0; + if (!mode.Bonds.IsEmpty()) + for (unsigned c = 0;; c++) + { + if (c == mode.Methods.Size()) + return S_OK; + if (!mode.IsThereBond_to_Coder(c)) + { + nextCoder = c; + break; + } + } +#endif } else { + // we insert new filter method: + CMethodFull &m = mode.Methods.InsertNew(0); // 0 == index of new inserted item + { + // we move all coder indexes in bonds up for 1 position: + FOR_VECTOR (k, mode.Bonds) + { + CBond2 &bond = mode.Bonds[k]; + bond.InCoder++; + bond.OutCoder++; + } + } + if (useBcj2) + { + GetMethodFull(k_BCJ2, 4, m); + return AddBcj2Methods(mode); + } + GetMethodFull(filterMode.Id, 1, m); + if (filterMode.Id == k_Delta) m.AddProp32(NCoderPropID::kDefaultProp, filterMode.Delta); - else if (filterMode.Id == k_ARM64) + else if (filterMode.Id == k_ARM64 + || filterMode.Id == k_RISCV) { // if (filterMode.Offset != 0) - m.AddProp32( - NCoderPropID::kDefaultProp, - // NCoderPropID::kBranchOffset, - filterMode.Offset); + m.AddProp32( + NCoderPropID::kDefaultProp, + // NCoderPropID::kBranchOffset, + filterMode.Offset); } - res = AddFilterBond(mode); - - int alignBits = -1; + + nextCoder = 1; + if (!mode.Bonds.IsEmpty()) { - const UInt32 delta = filterMode.Delta; - if (delta == 0 || delta > 16) - { - // if (delta == 0) alignBits = GetAlignForFilterMethod(filterMode.Id); - } - else if ((delta & ((1 << 4) - 1)) == 0) alignBits = 4; - else if ((delta & ((1 << 3) - 1)) == 0) alignBits = 3; - else if ((delta & ((1 << 2) - 1)) == 0) alignBits = 2; - else if ((delta & ((1 << 1) - 1)) == 0) alignBits = 1; - // else alignBits = 0; - /* alignBits=0 is default mode for lzma/lzma2. - So we don't set alignBits=0 here. */ + RINOK(AddBondForFilter(mode)) + nextCoder = mode.Bonds.Back().InCoder; } - if (res == S_OK && alignBits > 0) + } + + if (nextCoder >= mode.Methods.Size()) + { + // we don't expect that case, if there was non-filter method. + // but we return S_OK to support filter-only case. + return S_OK; + } + + int alignBits = -1; + { + const UInt32 delta = filterMode.Delta; + if (delta == 0 || delta > 16) { - unsigned nextCoder = 1; - if (!mode.Bonds.IsEmpty()) - { - nextCoder = mode.Bonds.Back().InCoder; - } - if (nextCoder < mode.Methods.Size()) - { - CMethodFull &nextMethod = mode.Methods[nextCoder]; - if (nextMethod.Id == k_LZMA || nextMethod.Id == k_LZMA2) - { - if (!nextMethod.Are_Lzma_Model_Props_Defined()) - { - if (alignBits != 0) - { - if (alignBits > 2 || filterMode.Id == k_Delta) - nextMethod.AddProp32(NCoderPropID::kPosStateBits, (unsigned)alignBits); - unsigned lc = 0; - if (alignBits < 3) - lc = (unsigned)(3 - alignBits); - nextMethod.AddProp32(NCoderPropID::kLitContextBits, lc); - nextMethod.AddProp32(NCoderPropID::kLitPosBits, (unsigned)alignBits); - } - } - } - } + // if (delta == 0) alignBits = GetAlignForFilterMethod(filterMode.Id); } + else if ((delta & ((1 << 4) - 1)) == 0) alignBits = 4; + else if ((delta & ((1 << 3) - 1)) == 0) alignBits = 3; + else if ((delta & ((1 << 2) - 1)) == 0) alignBits = 2; + else if ((delta & ((1 << 1) - 1)) == 0) alignBits = 1; + // else alignBits = 0; + /* alignBits=0 is default mode for lzma/lzma2. + So we don't set alignBits=0 here. */ } - - return res; + if (alignBits <= 0) + return S_OK; + // (alignBits > 0) + CMethodFull &nextMethod = mode.Methods[nextCoder]; + if (nextMethod.Id == k_LZMA || nextMethod.Id == k_LZMA2) + if (!nextMethod.Are_Lzma_Model_Props_Defined()) + { + if (alignBits > 2 || filterMode.Id == k_Delta) + nextMethod.AddProp32(NCoderPropID::kPosStateBits, (unsigned)alignBits); + const unsigned lc = (alignBits < 3) ? (unsigned)(3 - alignBits) : 0u; + nextMethod.AddProp32(NCoderPropID::kLitContextBits, lc); + nextMethod.AddProp32(NCoderPropID::kLitPosBits, (unsigned)alignBits); + } + return S_OK; } @@ -1859,6 +1927,7 @@ HRESULT Update( CIntArr fileIndexToUpdateIndexMap; UInt64 complexity = 0; + bool isThere_UnknownSize = false; UInt64 inSizeForReduce2 = 0; #ifndef Z7_NO_CRYPTO @@ -1971,28 +2040,36 @@ HRESULT Update( UInt64 inSizeForReduce = 0; { - bool isSolid = (numSolidFiles > 1 && options.NumSolidBytes != 0); + const bool isSolid = (numSolidFiles > 1 && options.NumSolidBytes != 0); FOR_VECTOR (i, updateItems) { const CUpdateItem &ui = updateItems[i]; if (ui.NewData) { - complexity += ui.Size; - if (isSolid) - inSizeForReduce += ui.Size; - else if (inSizeForReduce < ui.Size) - inSizeForReduce = ui.Size; + if (ui.Size == (UInt64)(Int64)-1) + isThere_UnknownSize = true; + else + { + complexity += ui.Size; + if (isSolid) + inSizeForReduce += ui.Size; + else if (inSizeForReduce < ui.Size) + inSizeForReduce = ui.Size; + } } } } + if (isThere_UnknownSize) + inSizeForReduce = (UInt64)(Int64)-1; + else + RINOK(updateCallback->SetTotal(complexity)) + if (inSizeForReduce < inSizeForReduce2) - inSizeForReduce = inSizeForReduce2; + inSizeForReduce = inSizeForReduce2; - RINOK(updateCallback->SetTotal(complexity)) - CLocalProgress *lps = new CLocalProgress; - CMyComPtr progress = lps; + CMyComPtr2_Create lps; lps->Init(updateCallback, true); #ifndef Z7_ST @@ -2232,12 +2309,13 @@ HRESULT Update( CCompressionMethodMode method = *options.Method; { const HRESULT res = MakeExeMethod(method, filterMode, + // bcj2_IsAllowed: #ifdef Z7_ST false #else options.MaxFilter && options.MultiThreadMixer #endif - ); + , options.DisabledFilterIDs); RINOK(res) } @@ -2299,7 +2377,7 @@ HRESULT Update( const UInt64 packSize = db->GetFolderFullPackSize(folderIndex); RINOK(WriteRange(inStream, archive.SeqStream, - db->GetFolderStreamPos(folderIndex, 0), packSize, progress)) + db->GetFolderStreamPos(folderIndex, 0), packSize, lps)) lps->ProgressOffset += packSize; const unsigned folderIndex_New = newDatabase.Folders.Size(); @@ -2321,7 +2399,7 @@ HRESULT Update( size_t indexStart = db->FoToCoderUnpackSizes[folderIndex]; const size_t indexEnd = db->FoToCoderUnpackSizes[folderIndex + 1]; for (; indexStart < indexEnd; indexStart++) - newDatabase.CoderUnpackSizes.Add(db->CoderUnpackSizes[indexStart]); + newDatabase.CoderUnpackSizes.Add(db->CoderUnpackSizes.ConstData()[indexStart]); } else { @@ -2481,7 +2559,7 @@ HRESULT Update( sizeToEncode, // expectedDataSize newDatabase.Folders.AddNew(), // newDatabase.CoderUnpackSizes, curUnpackSize, - archive.SeqStream, newDatabase.PackSizes, progress); + archive.SeqStream, newDatabase.PackSizes, lps); if (encodeRes == k_My_HRESULT_CRC_ERROR) return E_FAIL; @@ -2680,8 +2758,7 @@ HRESULT Update( */ - CFolderInStream *inStreamSpec = new CFolderInStream; - CMyComPtr solidInStream(inStreamSpec); + CMyComPtr2_Create inStreamSpec; // solidInStream; // inStreamSpec->_reportArcProp = reportArcProp; @@ -2702,13 +2779,13 @@ HRESULT Update( RINOK(encoder.Encode1( EXTERNAL_CODECS_LOC_VARS - solidInStream, + inStreamSpec, // NULL, &inSizeForReduce, expectedDataSize, // expected size newDatabase.Folders.AddNew(), // newDatabase.CoderUnpackSizes, curFolderUnpackSize, - archive.SeqStream, newDatabase.PackSizes, progress)) + archive.SeqStream, newDatabase.PackSizes, lps)) if (!inStreamSpec->WasFinished()) return E_FAIL; @@ -2970,7 +3047,7 @@ HRESULT Update( if (newDatabase.NumUnpackStreamsVector.Size() != numFolders || newDatabase.FolderUnpackCRCs.Defs.Size() > numFolders) return E_FAIL; - newDatabase.FolderUnpackCRCs.if_NonEmpty_FillResedue_with_false(numFolders); + newDatabase.FolderUnpackCRCs.if_NonEmpty_FillResidue_with_false(numFolders); } updateItems.ClearAndFree(); diff --git a/CPP/7zip/Archive/7z/7zUpdate.h b/CPP/7zip/Archive/7z/7zUpdate.h index de4117a..83a2774 100644 --- a/CPP/7zip/Archive/7z/7zUpdate.h +++ b/CPP/7zip/Archive/7z/7zUpdate.h @@ -111,6 +111,25 @@ struct CUpdateOptions CHeaderOptions HeaderOptions; + CUIntVector DisabledFilterIDs; + + void Add_DisabledFilter_for_id(UInt32 id, + const CUIntVector &enabledFilters) + { + if (enabledFilters.FindInSorted(id) < 0) + DisabledFilterIDs.AddToUniqueSorted(id); + } + + void SetFilterSupporting_ver_enabled_disabled( + UInt32 compatVer, + const CUIntVector &enabledFilters, + const CUIntVector &disabledFilters) + { + DisabledFilterIDs = disabledFilters; + if (compatVer < 2300) Add_DisabledFilter_for_id(k_ARM64, enabledFilters); + if (compatVer < 2402) Add_DisabledFilter_for_id(k_RISCV, enabledFilters); + } + CUpdateOptions(): Method(NULL), HeaderMethod(NULL), @@ -128,7 +147,9 @@ struct CUpdateOptions Need_MTime(false), Need_Attrib(false) // , Need_Crc(true) - {} + { + DisabledFilterIDs.Add(k_RISCV); + } }; HRESULT Update( diff --git a/CPP/7zip/Archive/ApfsHandler.cpp b/CPP/7zip/Archive/ApfsHandler.cpp index 5fb7e0a..62b00ed 100644 --- a/CPP/7zip/Archive/ApfsHandler.cpp +++ b/CPP/7zip/Archive/ApfsHandler.cpp @@ -54,31 +54,18 @@ namespace NArchive { namespace NApfs { -#define ValToHex(t) ((char)(((t) < 10) ? ('0' + (t)) : ('a' + ((t) - 10)))) - -static void ConvertByteToHex(unsigned val, char *s) -{ - unsigned t; - t = val >> 4; - s[0] = ValToHex(t); - t = val & 0xF; - s[1] = ValToHex(t); -} - struct CUuid { Byte Data[16]; void SetHex_To_str(char *s) const { - for (unsigned i = 0; i < 16; i++) - ConvertByteToHex(Data[i], s + i * 2); - s[32] = 0; + ConvertDataToHex_Lower(s, Data, sizeof(Data)); } void AddHexToString(UString &dest) const { - char temp[32 + 4]; + char temp[sizeof(Data) * 2 + 4]; SetHex_To_str(temp); dest += temp; } @@ -2151,7 +2138,7 @@ HRESULT CDatabase::ReadMap(UInt64 oid, bool noHeader, if (bti.Is_NOHEADER() != noHeader) return S_FALSE; // we don't allow volumes with big number of Keys - const UInt32 kNumItemsMax = k_VectorSizeMax; + const unsigned kNumItemsMax = k_VectorSizeMax; if (map.bti.node_count > kNumItemsMax) return S_FALSE; if (map.bti.key_count > kNumItemsMax) @@ -3759,7 +3746,7 @@ void CDatabase::GetItemPath(unsigned index, const CNode *inode, NWindows::NCOM:: #ifdef APFS_SHOW_ALT_STREAMS if (IsViDef(ref.AttrIndex) && inode) { - s += ':'; + s.Add_Colon(); Utf8Name_to_InterName(inode->Attrs[(unsigned)ref.AttrIndex].Name, s2); // s2 += "a\\b"; // for debug s += s2; @@ -4094,7 +4081,6 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, if (numItems == 0) return S_OK; UInt32 i; - { UInt64 totalSize = 0; for (i = 0; i < numItems; i++) @@ -4107,14 +4093,15 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, UInt64 currentTotalSize = 0, currentItemSize = 0; - CLocalProgress *lps = new CLocalProgress; - CMyComPtr progress = lps; + CMyComPtr2_Create lps; lps->Init(extractCallback, false); - NCompress::CCopyCoder *copyCoderSpec = new NCompress::CCopyCoder(); - CMyComPtr copyCoder = copyCoderSpec; + CMyComPtr2_Create copyCoder; - NHfs::CDecoder decoder; + // We don't know if zlib without Adler is allowed in APFS. + // But zlib without Adler is allowed in HFS. + // So here we allow apfs/zlib without Adler: + NHfs::CDecoder decoder(true); // IsAdlerOptional for (i = 0;; i++, currentTotalSize += currentItemSize) { @@ -4131,8 +4118,9 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, currentItemSize = GetSize(index); + int opRes; + { CMyComPtr realOutStream; - const Int32 askMode = testMode ? NExtract::NAskMode::kTest : NExtract::NAskMode::kExtract; @@ -4165,7 +4153,7 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, if (!testMode && !realOutStream) continue; RINOK(extractCallback->PrepareOperation(askMode)) - int opRes = NExtract::NOperationResult::kDataError; + opRes = NExtract::NOperationResult::kDataError; if (IsViDef(ref.NodeIndex)) { @@ -4221,38 +4209,38 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, CMyComPtr inStream; if (GetStream(index, &inStream) == S_OK && inStream) { - COutStreamWithHash *hashStreamSpec = NULL; - CMyComPtr hashStream; - + CMyComPtr2 hashStream; + if (vol.integrity.Is_SHA256()) { const int hashIndex = FindHashIndex_for_Item(index); if (hashIndex != -1) { - hashStreamSpec = new COutStreamWithHash; - hashStream = hashStreamSpec; - hashStreamSpec->SetStream(realOutStream); - hashStreamSpec->Init(&(vol.Hash_Vectors[(unsigned)hashIndex]), sb.block_size_Log); + hashStream.Create_if_Empty(); + hashStream->SetStream(realOutStream); + hashStream->Init(&(vol.Hash_Vectors[(unsigned)hashIndex]), sb.block_size_Log); } } - RINOK(copyCoder->Code(inStream, - hashStream ? hashStream : realOutStream, NULL, NULL, progress)) + RINOK(copyCoder.Interface()->Code(inStream, + hashStream.IsDefined() ? + hashStream.Interface() : + realOutStream.Interface(), + NULL, NULL, lps)) opRes = NExtract::NOperationResult::kDataError; - if (copyCoderSpec->TotalSize == currentItemSize) + if (copyCoder->TotalSize == currentItemSize) { opRes = NExtract::NOperationResult::kOK; - if (hashStream) - if (!hashStreamSpec->FinalCheck()) + if (hashStream.IsDefined()) + if (!hashStream->FinalCheck()) opRes = NExtract::NOperationResult::kCRCError; } - else if (copyCoderSpec->TotalSize < currentItemSize) + else if (copyCoder->TotalSize < currentItemSize) opRes = NExtract::NOperationResult::kUnexpectedEnd; } } } - - realOutStream.Release(); + } RINOK(extractCallback->SetOperationResult(opRes)) } return S_OK; @@ -4340,9 +4328,9 @@ Z7_COM7F_IMF(CHandler::GetStream(UInt32 index, ISequentialInStream **stream)) const CAttr &attr = inode.Attrs[(unsigned)attrIndex]; if (!attr.dstream_defined) { - CBufInStream *streamSpec = new CBufInStream; - CMyComPtr streamTemp = streamSpec; - streamSpec->Init(attr.Data, attr.Data.Size(), (IInArchive *)this); + CMyComPtr2 streamTemp; + streamTemp.Create_if_Empty(); + streamTemp->Init(attr.Data, attr.Data.Size(), (IInArchive *)this); *stream = streamTemp.Detach(); return S_OK; } @@ -4392,9 +4380,9 @@ HRESULT CDatabase::GetAttrStream(IInStream *apfsInStream, const CVol &vol, *stream = NULL; if (!attr.dstream_defined) { - CBufInStream *streamSpec = new CBufInStream; - CMyComPtr streamTemp = streamSpec; - streamSpec->Init(attr.Data, attr.Data.Size(), (IInArchive *)this); + CMyComPtr2 streamTemp; + streamTemp.Create_if_Empty(); + streamTemp->Init(attr.Data, attr.Data.Size(), (IInArchive *)this); *stream = streamTemp.Detach(); return S_OK; } @@ -4427,8 +4415,8 @@ HRESULT CDatabase::GetStream2( const CRecordVector *extents, UInt64 rem, ISequentialInStream **stream) { - CExtentsStream *extentStreamSpec = new CExtentsStream(); - CMyComPtr extentStream = extentStreamSpec; + CMyComPtr2 extentStream; + extentStream.Create_if_Empty(); UInt64 virt = 0; FOR_VECTOR (i, *extents) @@ -4452,7 +4440,7 @@ HRESULT CDatabase::GetStream2( se.Virt = virt; virt += cur; rem -= cur; - extentStreamSpec->Extents.Add(se); + extentStream->Extents.Add(se); if (rem == 0) if (i != extents->Size() - 1) return S_FALSE; @@ -4464,9 +4452,9 @@ HRESULT CDatabase::GetStream2( CSeekExtent se; se.Phy = 0; se.Virt = virt; - extentStreamSpec->Extents.Add(se); - extentStreamSpec->Stream = apfsInStream; - extentStreamSpec->Init(); + extentStream->Extents.Add(se); + extentStream->Stream = apfsInStream; + extentStream->Init(); *stream = extentStream.Detach(); return S_OK; } diff --git a/CPP/7zip/Archive/ApmHandler.cpp b/CPP/7zip/Archive/ApmHandler.cpp index 6770301..56d9b6e 100644 --- a/CPP/7zip/Archive/ApmHandler.cpp +++ b/CPP/7zip/Archive/ApmHandler.cpp @@ -5,36 +5,64 @@ #include "../../../C/CpuArch.h" #include "../../Common/ComTry.h" -#include "../../Common/Defs.h" #include "../../Windows/PropVariant.h" +#include "../../Windows/PropVariantUtils.h" #include "../Common/RegisterArc.h" #include "../Common/StreamUtils.h" #include "HandlerCont.h" -#define Get16(p) GetBe16(p) +// #define Get16(p) GetBe16(p) #define Get32(p) GetBe32(p) using namespace NWindows; namespace NArchive { + +namespace NDmg { + const char *Find_Apple_FS_Ext(const AString &name); + bool Is_Apple_FS_Or_Unknown(const AString &name); +} + namespace NApm { static const Byte kSig0 = 'E'; static const Byte kSig1 = 'R'; +static const CUInt32PCharPair k_Flags[] = +{ + { 0, "VALID" }, + { 1, "ALLOCATED" }, + { 2, "IN_USE" }, + { 3, "BOOTABLE" }, + { 4, "READABLE" }, + { 5, "WRITABLE" }, + { 6, "OS_PIC_CODE" }, + // { 7, "OS_SPECIFIC_2" }, // "Unused" + // { 8, "ChainCompatible" }, // "OS_SPECIFIC_1" + // { 9, "RealDeviceDriver" }, + // { 10, "CanChainToNext" }, + { 30, "MOUNTED_AT_STARTUP" }, + { 31, "STARTUP" } +}; + +#define DPME_FLAGS_VALID (1u << 0) +#define DPME_FLAGS_ALLOCATED (1u << 1) + +static const unsigned k_Str_Size = 32; + struct CItem { UInt32 StartBlock; UInt32 NumBlocks; - char Name[32]; - char Type[32]; + UInt32 Flags; // pmPartStatus + char Name[k_Str_Size]; + char Type[k_Str_Size]; /* UInt32 DataStartBlock; UInt32 NumDataBlocks; - UInt32 Status; UInt32 BootStartBlock; UInt32 BootSize; UInt32 BootAddr; @@ -43,25 +71,28 @@ struct CItem char Processor[16]; */ + bool Is_Valid_and_Allocated() const + { return (Flags & (DPME_FLAGS_VALID | DPME_FLAGS_ALLOCATED)) != 0; } + bool Parse(const Byte *p, UInt32 &numBlocksInMap) { numBlocksInMap = Get32(p + 4); StartBlock = Get32(p + 8); - NumBlocks = Get32(p + 0xC); - memcpy(Name, p + 0x10, 32); - memcpy(Type, p + 0x30, 32); - if (p[0] != 0x50 || p[1] != 0x4D || p[2] != 0 || p[3] != 0) + NumBlocks = Get32(p + 0xc); + Flags = Get32(p + 0x58); + memcpy(Name, p + 0x10, k_Str_Size); + memcpy(Type, p + 0x30, k_Str_Size); + if (GetUi32(p) != 0x4d50) // "PM" return false; /* DataStartBlock = Get32(p + 0x50); NumDataBlocks = Get32(p + 0x54); - Status = Get32(p + 0x58); - BootStartBlock = Get32(p + 0x5C); + BootStartBlock = Get32(p + 0x5c); BootSize = Get32(p + 0x60); BootAddr = Get32(p + 0x64); if (Get32(p + 0x68) != 0) return false; - BootEntry = Get32(p + 0x6C); + BootEntry = Get32(p + 0x6c); if (Get32(p + 0x70) != 0) return false; BootChecksum = Get32(p + 0x74); @@ -71,6 +102,7 @@ struct CItem } }; + Z7_class_CHandler_final: public CHandlerCont { Z7_IFACE_COM7_IMP(IInArchive_Cont) @@ -81,7 +113,6 @@ Z7_class_CHandler_final: public CHandlerCont UInt64 _phySize; bool _isArc; - HRESULT ReadTables(IInStream *stream); UInt64 BlocksToBytes(UInt32 i) const { return (UInt64)i << _blockSizeLog; } virtual int GetItem_ExtractInfo(UInt32 index, UInt64 &pos, UInt64 &size) const Z7_override @@ -95,47 +126,58 @@ Z7_class_CHandler_final: public CHandlerCont static const UInt32 kSectorSize = 512; +// we support only 4 cluster sizes: 512, 1024, 2048, 4096 */ + API_FUNC_static_IsArc IsArc_Apm(const Byte *p, size_t size) { if (size < kSectorSize) return k_IsArc_Res_NEED_MORE; - if (p[0] != kSig0 || p[1] != kSig1) + if (GetUi64(p + 8) != 0) + return k_IsArc_Res_NO; + UInt32 v = GetUi32(p); // we read as little-endian + v ^= (kSig0 | (unsigned)kSig1 << 8); + if ((v & ~((UInt32)0xf << 17))) return k_IsArc_Res_NO; - unsigned i; - for (i = 8; i < 16; i++) - if (p[i] != 0) - return k_IsArc_Res_NO; - UInt32 blockSize = Get16(p + 2); - for (i = 9; ((UInt32)1 << i) != blockSize; i++) - if (i >= 12) - return k_IsArc_Res_NO; - return k_IsArc_Res_YES; + if ((0x116u >> (v >> 17)) & 1) + return k_IsArc_Res_YES; + return k_IsArc_Res_NO; } } -HRESULT CHandler::ReadTables(IInStream *stream) +Z7_COM7F_IMF(CHandler::Open(IInStream *stream, const UInt64 *, IArchiveOpenCallback * /* callback */)) { + COM_TRY_BEGIN + Close(); + Byte buf[kSectorSize]; + unsigned numSectors_in_Cluster; { RINOK(ReadStream_FALSE(stream, buf, kSectorSize)) - if (buf[0] != kSig0 || buf[1] != kSig1) + if (GetUi64(buf + 8) != 0) return S_FALSE; - UInt32 blockSize = Get16(buf + 2); - unsigned i; - for (i = 9; ((UInt32)1 << i) != blockSize; i++) - if (i >= 12) - return S_FALSE; - _blockSizeLog = i; - _numBlocks = Get32(buf + 4); - for (i = 8; i < 16; i++) - if (buf[i] != 0) - return S_FALSE; + UInt32 v = GetUi32(buf); // we read as little-endian + v ^= (kSig0 | (unsigned)kSig1 << 8); + if ((v & ~((UInt32)0xf << 17))) + return S_FALSE; + v >>= 16; + if (v == 0) + return S_FALSE; + if (v & (v - 1)) + return S_FALSE; + const unsigned a = (0x30210u >> v) & 3; + // a = 0; // for debug + numSectors_in_Cluster = 1u << a; + _blockSizeLog = 9 + a; } - unsigned numSkips = (unsigned)1 << (_blockSizeLog - 9); - for (unsigned j = 1; j < numSkips; j++) + UInt32 numBlocks = Get32(buf + 4); + _numBlocks = numBlocks; + { - RINOK(ReadStream_FALSE(stream, buf, kSectorSize)) + for (unsigned k = numSectors_in_Cluster; --k != 0;) + { + RINOK(ReadStream_FALSE(stream, buf, kSectorSize)) + } } UInt32 numBlocksInMap = 0; @@ -152,19 +194,20 @@ HRESULT CHandler::ReadTables(IInStream *stream) if (i == 0) { numBlocksInMap = numBlocksInMap2; - if (numBlocksInMap > (1 << 8)) + if (numBlocksInMap > (1 << 8) || numBlocksInMap == 0) return S_FALSE; } else if (numBlocksInMap2 != numBlocksInMap) return S_FALSE; - UInt32 finish = item.StartBlock + item.NumBlocks; + const UInt32 finish = item.StartBlock + item.NumBlocks; if (finish < item.StartBlock) return S_FALSE; - _numBlocks = MyMax(_numBlocks, finish); + if (numBlocks < finish) + numBlocks = finish; _items.Add(item); - for (unsigned j = 1; j < numSkips; j++) + for (unsigned k = numSectors_in_Cluster; --k != 0;) { RINOK(ReadStream_FALSE(stream, buf, kSectorSize)) } @@ -172,21 +215,15 @@ HRESULT CHandler::ReadTables(IInStream *stream) break; } - _phySize = BlocksToBytes(_numBlocks); + _phySize = BlocksToBytes(numBlocks); _isArc = true; - return S_OK; -} - -Z7_COM7F_IMF(CHandler::Open(IInStream *stream, const UInt64 *, IArchiveOpenCallback * /* callback */)) -{ - COM_TRY_BEGIN - Close(); - RINOK(ReadTables(stream)) _stream = stream; + return S_OK; COM_TRY_END } + Z7_COM7F_IMF(CHandler::Close()) { _isArc = false; @@ -196,16 +233,19 @@ Z7_COM7F_IMF(CHandler::Close()) return S_OK; } + static const Byte kProps[] = { kpidPath, kpidSize, - kpidOffset + kpidOffset, + kpidCharacts }; static const Byte kArcProps[] = { - kpidClusterSize + kpidClusterSize, + kpidNumBlocks }; IMP_IInArchive_Props @@ -214,8 +254,7 @@ IMP_IInArchive_ArcProps static AString GetString(const char *s) { AString res; - for (unsigned i = 0; i < 32 && s[i] != 0; i++) - res += s[i]; + res.SetFrom_CalcLen(s, k_Str_Size); return res; } @@ -230,11 +269,13 @@ Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) int mainIndex = -1; FOR_VECTOR (i, _items) { - AString s (GetString(_items[i].Type)); - if (s != "Apple_Free" && - s != "Apple_partition_map") + const CItem &item = _items[i]; + if (!item.Is_Valid_and_Allocated()) + continue; + AString s (GetString(item.Type)); + if (NDmg::Is_Apple_FS_Or_Unknown(s)) { - if (mainIndex >= 0) + if (mainIndex != -1) { mainIndex = -1; break; @@ -242,12 +283,13 @@ Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) mainIndex = (int)i; } } - if (mainIndex >= 0) + if (mainIndex != -1) prop = (UInt32)(Int32)mainIndex; break; } case kpidClusterSize: prop = (UInt32)1 << _blockSizeLog; break; case kpidPhySize: prop = _phySize; break; + case kpidNumBlocks: prop = _numBlocks; break; case kpidErrorFlags: { @@ -281,8 +323,11 @@ Z7_COM7F_IMF(CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val if (s.IsEmpty()) s.Add_UInt32(index); AString type (GetString(item.Type)); - if (type == "Apple_HFS") - type = "hfs"; + { + const char *ext = NDmg::Find_Apple_FS_Ext(type); + if (ext) + type = ext; + } if (!type.IsEmpty()) { s.Add_Dot(); @@ -296,6 +341,7 @@ Z7_COM7F_IMF(CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val prop = BlocksToBytes(item.NumBlocks); break; case kpidOffset: prop = BlocksToBytes(item.StartBlock); break; + case kpidCharacts: FLAGS_TO_PROP(k_Flags, item.Flags, prop); break; } prop.Detach(value); return S_OK; diff --git a/CPP/7zip/Archive/ArHandler.cpp b/CPP/7zip/Archive/ArHandler.cpp index 07ecec6..95e4719 100644 --- a/CPP/7zip/Archive/ArHandler.cpp +++ b/CPP/7zip/Archive/ArHandler.cpp @@ -274,18 +274,18 @@ HRESULT CInArchive::GetNextItem(CItem &item, bool &filled) Z7_CLASS_IMP_CHandler_IInArchive_1( IInArchiveGetStream ) + bool _isArc; CObjectVector _items; CMyComPtr _stream; - Int32 _mainSubfile; UInt64 _phySize; + Int32 _mainSubfile; EType _type; ESubType _subType; int _longNames_FileIndex; - AString _libFiles[2]; unsigned _numLibFiles; AString _errorMessage; - bool _isArc; + AString _libFiles[2]; void UpdateErrorMessage(const char *s); @@ -299,7 +299,7 @@ Z7_CLASS_IMP_CHandler_IInArchive_1( void CHandler::UpdateErrorMessage(const char *s) { if (!_errorMessage.IsEmpty()) - _errorMessage += '\n'; + _errorMessage.Add_LF(); _errorMessage += s; } @@ -437,8 +437,8 @@ HRESULT CHandler::AddFunc(UInt32 offset, const Byte *data, size_t size, size_t & s.DeleteBack(); s += " "; s += (const char *)(data + pos); - s += (char)0xD; - s += (char)0xA; + // s.Add_Char((char)0xD); + s.Add_LF(); pos = i; return S_OK; } @@ -473,7 +473,7 @@ HRESULT CHandler::ParseLibSymbols(IInStream *stream, unsigned fileIndex) if (size - pos < tableSize || (tableSize & 7) != 0) continue; size_t namesStart = pos + tableSize; - const UInt32 namesSize = Get32(p + namesStart, be); + const UInt32 namesSize = Get32(p.ConstData() + namesStart, be); namesStart += 4; if (namesStart > size || namesStart + namesSize != size) continue; @@ -582,7 +582,7 @@ Z7_COM7F_IMF(CHandler::Open(IInStream *stream, arc.SkipData(item.Size); if (callback && (_items.Size() & 0xFF) == 0) { - UInt64 numFiles = _items.Size(); + const UInt64 numFiles = _items.Size(); RINOK(callback->SetCompleted(&numFiles, &arc.Position)) } } @@ -762,21 +762,21 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, UInt64 currentTotalSize = 0; - NCompress::CCopyCoder *copyCoderSpec = new NCompress::CCopyCoder(); - CMyComPtr copyCoder = copyCoderSpec; - - CLocalProgress *lps = new CLocalProgress; - CMyComPtr progress = lps; + CMyComPtr2_Create copyCoder; + CMyComPtr2_Create lps; lps->Init(extractCallback, false); - CLimitedSequentialInStream *streamSpec = new CLimitedSequentialInStream; - CMyComPtr inStream(streamSpec); - streamSpec->SetStream(_stream); + CMyComPtr2_Create inStream; + inStream->SetStream(_stream); - for (i = 0; i < numItems; i++) + for (i = 0;; i++) { lps->InSize = lps->OutSize = currentTotalSize; RINOK(lps->SetCur()) + if (i >= numItems) + break; + Int32 opRes; + { CMyComPtr realOutStream; const Int32 askMode = testMode ? NExtract::NAskMode::kTest : @@ -795,7 +795,7 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, RINOK(extractCallback->SetOperationResult(NExtract::NOperationResult::kOK)) continue; } - bool isOk = true; + opRes = NExtract::NOperationResult::kOK; if (item.TextFileIndex >= 0) { const AString &f = _libFiles[(unsigned)item.TextFileIndex]; @@ -805,14 +805,13 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, else { RINOK(InStream_SeekSet(_stream, item.GetDataPos())) - streamSpec->Init(item.Size); - RINOK(copyCoder->Code(inStream, realOutStream, NULL, NULL, progress)) - isOk = (copyCoderSpec->TotalSize == item.Size); + inStream->Init(item.Size); + RINOK(copyCoder.Interface()->Code(inStream, realOutStream, NULL, NULL, lps)) + if (copyCoder->TotalSize != item.Size) + opRes = NExtract::NOperationResult::kDataError; } - realOutStream.Release(); - RINOK(extractCallback->SetOperationResult(isOk ? - NExtract::NOperationResult::kOK: - NExtract::NOperationResult::kDataError)) + } + RINOK(extractCallback->SetOperationResult(opRes)) } return S_OK; COM_TRY_END diff --git a/CPP/7zip/Archive/ArjHandler.cpp b/CPP/7zip/Archive/ArjHandler.cpp index 2f982c4..1d641e4 100644 --- a/CPP/7zip/Archive/ArjHandler.cpp +++ b/CPP/7zip/Archive/ArjHandler.cpp @@ -4,6 +4,7 @@ #include "../../../C/CpuArch.h" +#include "../../Common/AutoPtr.h" #include "../../Common/ComTry.h" #include "../../Common/StringConvert.h" @@ -28,16 +29,13 @@ namespace NArj { namespace NDecoder { static const unsigned kMatchMinLen = 3; - static const UInt32 kWindowSize = 1 << 15; // must be >= (1 << 14) - -Z7_CLASS_IMP_NOQIB_1( - CCoder - , ICompressCoder -) +class CCoder +{ CLzOutWindow _outWindow; NBitm::CDecoder _inBitStream; + // bool FinishMode; class CCoderReleaser { @@ -49,19 +47,20 @@ Z7_CLASS_IMP_NOQIB_1( }; friend class CCoderReleaser; - HRESULT CodeReal(UInt64 outSize, ICompressProgressInfo *progress); + HRESULT CodeReal(UInt32 outSize, ICompressProgressInfo *progress); public: - bool FinishMode; - CCoder(): FinishMode(false) {} + // CCoder(): FinishMode(true) {} UInt64 GetInputProcessedSize() const { return _inBitStream.GetProcessedSize(); } + HRESULT Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, + UInt32 outSize, ICompressProgressInfo *progress); }; -HRESULT CCoder::CodeReal(UInt64 rem, ICompressProgressInfo *progress) +HRESULT CCoder::CodeReal(UInt32 rem, ICompressProgressInfo *progress) { const UInt32 kStep = 1 << 20; - UInt64 next = 0; + UInt32 next = 0; if (rem > kStep && progress) next = rem - kStep; @@ -71,9 +70,8 @@ HRESULT CCoder::CodeReal(UInt64 rem, ICompressProgressInfo *progress) { if (_inBitStream.ExtraBitsWereRead()) return S_FALSE; - - UInt64 packSize = _inBitStream.GetProcessedSize(); - UInt64 pos = _outWindow.GetProcessedSize(); + const UInt64 packSize = _inBitStream.GetProcessedSize(); + const UInt64 pos = _outWindow.GetProcessedSize(); RINOK(progress->SetRatioInfo(&packSize, &pos)) next = 0; if (rem > kStep) @@ -81,12 +79,11 @@ HRESULT CCoder::CodeReal(UInt64 rem, ICompressProgressInfo *progress) } UInt32 len; - { const unsigned kNumBits = 7 + 7; - UInt32 val = _inBitStream.GetValue(kNumBits); + const UInt32 val = _inBitStream.GetValue(kNumBits); - if ((val & (1 << (kNumBits - 1))) == 0) + if ((val & (1u << (kNumBits - 1))) == 0) { _outWindow.PutByte((Byte)(val >> 5)); _inBitStream.MovePos(1 + 8); @@ -94,27 +91,24 @@ HRESULT CCoder::CodeReal(UInt64 rem, ICompressProgressInfo *progress) continue; } - UInt32 mask = 1 << (kNumBits - 2); unsigned w; - - for (w = 1; w < 7; w++, mask >>= 1) - if ((val & mask) == 0) - break; - - unsigned readBits = (w != 7 ? 1 : 0); - readBits += w + w; - len = (1 << w) - 1 + kMatchMinLen - 1 + - (((val >> (kNumBits - readBits)) & ((1 << w) - 1))); + { + UInt32 flag = (UInt32)1 << (kNumBits - 2); + for (w = 1; w < 7; w++, flag >>= 1) + if ((val & flag) == 0) + break; + } + const unsigned readBits = (w != 7 ? 1 : 0) + w * 2; + const UInt32 mask = ((UInt32)1 << w) - 1; + len = mask + kMatchMinLen - 1 + + ((val >> (kNumBits - readBits)) & mask); _inBitStream.MovePos(readBits); } - { const unsigned kNumBits = 4 + 13; - UInt32 val = _inBitStream.GetValue(kNumBits); - + const UInt32 val = _inBitStream.GetValue(kNumBits); unsigned readBits = 1; unsigned w; - if ((val & ((UInt32)1 << 16)) == 0) w = 9; else if ((val & ((UInt32)1 << 15)) == 0) w = 10; else if ((val & ((UInt32)1 << 14)) == 0) w = 11; @@ -122,61 +116,50 @@ HRESULT CCoder::CodeReal(UInt64 rem, ICompressProgressInfo *progress) else { w = 13; readBits = 0; } readBits += w + w - 9; - - UInt32 dist = ((UInt32)1 << w) - (1 << 9) + + const UInt32 dist = ((UInt32)1 << w) - (1 << 9) + (((val >> (kNumBits - readBits)) & ((1 << w) - 1))); _inBitStream.MovePos(readBits); - if (len > rem) - len = (UInt32)rem; - + { + // if (FinishMode) + return S_FALSE; + // else len = (UInt32)rem; + } if (!_outWindow.CopyBlock(dist, len)) return S_FALSE; rem -= len; } } - if (FinishMode) + // if (FinishMode) { if (_inBitStream.ReadAlignBits() != 0) return S_FALSE; } - if (_inBitStream.ExtraBitsWereRead()) return S_FALSE; - return S_OK; } - -Z7_COM7F_IMF(CCoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, - const UInt64 * /* inSize */, const UInt64 *outSize, ICompressProgressInfo *progress)) +HRESULT CCoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, + UInt32 outSize, ICompressProgressInfo *progress) { try { - if (!outSize) - return E_INVALIDARG; - if (!_outWindow.Create(kWindowSize)) return E_OUTOFMEMORY; if (!_inBitStream.Create(1 << 17)) return E_OUTOFMEMORY; - _outWindow.SetStream(outStream); _outWindow.Init(false); _inBitStream.SetStream(inStream); _inBitStream.Init(); - - CCoderReleaser coderReleaser(this); - HRESULT res; { - res = CodeReal(*outSize, progress); - if (res != S_OK) - return res; + CCoderReleaser coderReleaser(this); + RINOK(CodeReal(outSize, progress)) + coderReleaser.Disable(); } - - coderReleaser.Disable(); return _outWindow.Flush(); } catch(const CInBufferException &e) { return e.ErrorCode; } @@ -858,74 +841,70 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, totalUnpacked += item.Size; // totalPacked += item.PackSize; } - extractCallback->SetTotal(totalUnpacked); + RINOK(extractCallback->SetTotal(totalUnpacked)) totalUnpacked = totalPacked = 0; - UInt64 curUnpacked, curPacked; + UInt32 curUnpacked, curPacked; - NCompress::NLzh::NDecoder::CCoder *lzhDecoderSpec = NULL; - CMyComPtr lzhDecoder; - - NCompress::NArj::NDecoder::CCoder *arjDecoderSpec = NULL; - CMyComPtr arjDecoder; - - NCompress::CCopyCoder *copyCoderSpec = new NCompress::CCopyCoder(); - CMyComPtr copyCoder = copyCoderSpec; - - CLocalProgress *lps = new CLocalProgress; - CMyComPtr progress = lps; + CMyComPtr2_Create lps; lps->Init(extractCallback, false); - - CLimitedSequentialInStream *inStreamSpec = new CLimitedSequentialInStream; - CMyComPtr inStream(inStreamSpec); - inStreamSpec->SetStream(_stream); - - for (i = 0; i < numItems; i++, totalUnpacked += curUnpacked, totalPacked += curPacked) + CMyUniquePtr lzhDecoder; + CMyUniquePtr arjDecoder; + CMyComPtr2_Create copyCoder; + CMyComPtr2_Create inStream; + inStream->SetStream(_stream); + + for (i = 0;; i++, + totalUnpacked += curUnpacked, + totalPacked += curPacked) { lps->InSize = totalPacked; lps->OutSize = totalUnpacked; RINOK(lps->SetCur()) + if (i >= numItems) + break; curUnpacked = curPacked = 0; - CMyComPtr realOutStream; - const Int32 askMode = testMode ? - NExtract::NAskMode::kTest : - NExtract::NAskMode::kExtract; - const UInt32 index = allFilesMode ? i : indices[i]; - const CItem &item = _items[index]; - RINOK(extractCallback->GetStream(index, &realOutStream, askMode)) - - if (item.IsDir()) + Int32 opRes; { - // if (!testMode) + CMyComPtr realOutStream; + const Int32 askMode = testMode ? + NExtract::NAskMode::kTest : + NExtract::NAskMode::kExtract; + const UInt32 index = allFilesMode ? i : indices[i]; + const CItem &item = _items[index]; + RINOK(extractCallback->GetStream(index, &realOutStream, askMode)) + + if (item.IsDir()) { - RINOK(extractCallback->PrepareOperation(askMode)) - RINOK(extractCallback->SetOperationResult(NExtract::NOperationResult::kOK)) + // if (!testMode) + { + RINOK(extractCallback->PrepareOperation(askMode)) + // realOutStream.Release(); + RINOK(extractCallback->SetOperationResult(NExtract::NOperationResult::kOK)) + } + continue; } - continue; - } - - if (!testMode && !realOutStream) - continue; - - RINOK(extractCallback->PrepareOperation(askMode)) - curUnpacked = item.Size; - curPacked = item.PackSize; - - { - COutStreamWithCRC *outStreamSpec = new COutStreamWithCRC; - CMyComPtr outStream(outStreamSpec); - outStreamSpec->SetStream(realOutStream); - realOutStream.Release(); - outStreamSpec->Init(); + + if (!testMode && !realOutStream) + continue; + + RINOK(extractCallback->PrepareOperation(askMode)) + curUnpacked = item.Size; + curPacked = item.PackSize; + + CMyComPtr2_Create outStream; + outStream->SetStream(realOutStream); + // realOutStream.Release(); + outStream->Init(); - inStreamSpec->Init(item.PackSize); + inStream->Init(item.PackSize); RINOK(InStream_SeekSet(_stream, item.DataPosition)) HRESULT result = S_OK; - Int32 opRes = NExtract::NOperationResult::kOK; + opRes = NExtract::NOperationResult::kOK; if (item.IsEncrypted()) opRes = NExtract::NOperationResult::kUnsupportedMethod; @@ -935,8 +914,8 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, { case NCompressionMethod::kStored: { - result = copyCoder->Code(inStream, outStream, NULL, NULL, progress); - if (result == S_OK && copyCoderSpec->TotalSize != item.PackSize) + result = copyCoder.Interface()->Code(inStream, outStream, NULL, NULL, lps); + if (result == S_OK && copyCoder->TotalSize != item.PackSize) result = S_FALSE; break; } @@ -944,29 +923,21 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, case NCompressionMethod::kCompressed1b: case NCompressionMethod::kCompressed1c: { - if (!lzhDecoder) - { - lzhDecoderSpec = new NCompress::NLzh::NDecoder::CCoder; - lzhDecoder = lzhDecoderSpec; - } - lzhDecoderSpec->FinishMode = true; + lzhDecoder.Create_if_Empty(); + // lzhDecoder->FinishMode = true; const UInt32 kHistorySize = 26624; - lzhDecoderSpec->SetDictSize(kHistorySize); - result = lzhDecoder->Code(inStream, outStream, NULL, &curUnpacked, progress); - if (result == S_OK && lzhDecoderSpec->GetInputProcessedSize() != item.PackSize) + lzhDecoder->SetDictSize(kHistorySize); + result = lzhDecoder->Code(inStream, outStream, curUnpacked, lps); + if (result == S_OK && lzhDecoder->GetInputProcessedSize() != item.PackSize) result = S_FALSE; break; } case NCompressionMethod::kCompressed2: { - if (!arjDecoder) - { - arjDecoderSpec = new NCompress::NArj::NDecoder::CCoder; - arjDecoder = arjDecoderSpec; - } - arjDecoderSpec->FinishMode = true; - result = arjDecoder->Code(inStream, outStream, NULL, &curUnpacked, progress); - if (result == S_OK && arjDecoderSpec->GetInputProcessedSize() != item.PackSize) + arjDecoder.Create_if_Empty(); + // arjDecoderSpec->FinishMode = true; + result = arjDecoder->Code(inStream, outStream, curUnpacked, lps); + if (result == S_OK && arjDecoder->GetInputProcessedSize() != item.PackSize) result = S_FALSE; break; } @@ -982,15 +953,13 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, else { RINOK(result) - opRes = (outStreamSpec->GetCRC() == item.FileCRC) ? + opRes = (outStream->GetCRC() == item.FileCRC) ? NExtract::NOperationResult::kOK: NExtract::NOperationResult::kCRCError; } } - - outStream.Release(); - RINOK(extractCallback->SetOperationResult(opRes)) } + RINOK(extractCallback->SetOperationResult(opRes)) } return S_OK; diff --git a/CPP/7zip/Archive/AvbHandler.cpp b/CPP/7zip/Archive/AvbHandler.cpp new file mode 100644 index 0000000..4359fe8 --- /dev/null +++ b/CPP/7zip/Archive/AvbHandler.cpp @@ -0,0 +1,596 @@ +// AvbHandler.cpp + +#include "StdAfx.h" + +#include "../../../C/CpuArch.h" + +#include "../../Common/ComTry.h" +#include "../../Common/MyBuffer.h" + +#include "../../Windows/PropVariant.h" + +#include "../Common/RegisterArc.h" +#include "../Common/StreamUtils.h" + +#include "HandlerCont.h" + +#define Get32(p) GetBe32(p) +#define Get64(p) GetBe64(p) + +#define G32(_offs_, dest) dest = Get32(p + (_offs_)) +#define G64(_offs_, dest) dest = Get64(p + (_offs_)) + +using namespace NWindows; + +namespace NArchive { + +namespace NExt { +API_FUNC_IsArc IsArc_Ext_PhySize(const Byte *p, size_t size, UInt64 *phySize); +} + +namespace NAvb { + +static void AddNameToString(AString &s, const Byte *name, unsigned size, bool strictConvert) +{ + for (unsigned i = 0; i < size; i++) + { + Byte c = name[i]; + if (c == 0) + return; + if (strictConvert && c < 32) + c = '_'; + s += (char)c; + } +} + +/* Maximum size of a vbmeta image - 64 KiB. */ +#define VBMETA_MAX_SIZE (64 * 1024) + +#define SIGNATURE { 'A', 'V', 'B', 'f', 0, 0, 0, 1 } + +static const unsigned k_SignatureSize = 8; +static const Byte k_Signature[k_SignatureSize] = SIGNATURE; + +// #define AVB_FOOTER_MAGIC "AVBf" +// #define AVB_FOOTER_MAGIC_LEN 4 +/* The current footer version used - keep in sync with avbtool. */ +#define AVB_FOOTER_VERSION_MAJOR 1 + +/* The struct used as a footer used on partitions, used to find the + * AvbVBMetaImageHeader struct. This struct is always stored at the + * end of a partition. + */ +// #define AVB_FOOTER_SIZE 64 +static const unsigned kFooterSize = 64; + +struct CFooter +{ + /* 0: Four bytes equal to "AVBf" (AVB_FOOTER_MAGIC). */ + // Byte magic[AVB_FOOTER_MAGIC_LEN]; + /* 4: The major version of the footer struct. */ + UInt32 version_major; + /* 8: The minor version of the footer struct. */ + UInt32 version_minor; + + /* 12: The original size of the image on the partition. */ + UInt64 original_image_size; + + /* 20: The offset of the |AvbVBMetaImageHeader| struct. */ + UInt64 vbmeta_offset; + + /* 28: The size of the vbmeta block (header + auth + aux blocks). */ + UInt64 vbmeta_size; + + /* 36: Padding to ensure struct is size AVB_FOOTER_SIZE bytes. This + * must be set to zeroes. + */ + Byte reserved[28]; + + void Parse(const Byte *p) + { + G32 (4, version_major); + G32 (8, version_minor); + G64 (12, original_image_size); + G64 (20, vbmeta_offset); + G64 (28, vbmeta_size); + } +}; + + +/* Size of the vbmeta image header. */ +#define AVB_VBMETA_IMAGE_HEADER_SIZE 256 + +/* Magic for the vbmeta image header. */ +// #define AVB_MAGIC "AVB0" +// #define AVB_MAGIC_LEN 4 +/* Maximum size of the release string including the terminating NUL byte. */ +#define AVB_RELEASE_STRING_SIZE 48 + +struct AvbVBMetaImageHeader +{ + /* 0: Four bytes equal to "AVB0" (AVB_MAGIC). */ + // Byte magic[AVB_MAGIC_LEN]; + + /* 4: The major version of libavb required for this header. */ + UInt32 required_libavb_version_major; + /* 8: The minor version of libavb required for this header. */ + UInt32 required_libavb_version_minor; + + /* 12: The size of the signature block. */ + UInt64 authentication_data_block_size; + /* 20: The size of the auxiliary data block. */ + UInt64 auxiliary_data_block_size; + + /* 28: The verification algorithm used, see |AvbAlgorithmType| enum. */ + UInt32 algorithm_type; + + /* 32: Offset into the "Authentication data" block of hash data. */ + UInt64 hash_offset; + /* 40: Length of the hash data. */ + UInt64 hash_size; + + /* 48: Offset into the "Authentication data" block of signature data. */ + UInt64 signature_offset; + /* 56: Length of the signature data. */ + UInt64 signature_size; + + /* 64: Offset into the "Auxiliary data" block of public key data. */ + UInt64 public_key_offset; + /* 72: Length of the public key data. */ + UInt64 public_key_size; + + /* 80: Offset into the "Auxiliary data" block of public key metadata. */ + UInt64 public_key_metadata_offset; + /* 88: Length of the public key metadata. Must be set to zero if there + * is no public key metadata. + */ + UInt64 public_key_metadata_size; + + /* 96: Offset into the "Auxiliary data" block of descriptor data. */ + UInt64 descriptors_offset; + /* 104: Length of descriptor data. */ + UInt64 descriptors_size; + + /* 112: The rollback index which can be used to prevent rollback to + * older versions. + */ + UInt64 rollback_index; + + /* 120: Flags from the AvbVBMetaImageFlags enumeration. This must be + * set to zero if the vbmeta image is not a top-level image. + */ + UInt32 flags; + + /* 124: The location of the rollback index defined in this header. + * Only valid for the main vbmeta. For chained partitions, the rollback + * index location must be specified in the AvbChainPartitionDescriptor + * and this value must be set to 0. + */ + UInt32 rollback_index_location; + + /* 128: The release string from avbtool, e.g. "avbtool 1.0.0" or + * "avbtool 1.0.0 xyz_board Git-234abde89". Is guaranteed to be NUL + * terminated. Applications must not make assumptions about how this + * string is formatted. + */ + Byte release_string[AVB_RELEASE_STRING_SIZE]; + + /* 176: Padding to ensure struct is size AVB_VBMETA_IMAGE_HEADER_SIZE + * bytes. This must be set to zeroes. + */ + // Byte reserved[80]; + bool Parse(const Byte *p); +}; + +bool AvbVBMetaImageHeader::Parse(const Byte *p) +{ + // Byte magic[AVB_MAGIC_LEN]; + if (Get32(p) != 0x41564230) // "AVB0" + return false; + G32 (4, required_libavb_version_major); + if (required_libavb_version_major != AVB_FOOTER_VERSION_MAJOR) // "AVB0" + return false; + G32 (8, required_libavb_version_minor); + G64 (12, authentication_data_block_size); + G64 (20, auxiliary_data_block_size); + G32 (28, algorithm_type); + G64 (32, hash_offset); + G64 (40, hash_size); + G64 (48, signature_offset); + G64 (56, signature_size); + G64 (64, public_key_offset); + G64 (72, public_key_size); + G64 (80, public_key_metadata_offset); + G64 (88, public_key_metadata_size); + G64 (96, descriptors_offset); + G64 (104, descriptors_size); + G64 (112, rollback_index); + G32 (120, flags); + G32 (124, rollback_index_location); + memcpy(release_string, p + 128, AVB_RELEASE_STRING_SIZE); + + /* 176: Padding to ensure struct is size AVB_VBMETA_IMAGE_HEADER_SIZE + * bytes. This must be set to zeroes. + */ + // Byte reserved[80]; + return true; +} + + +static const unsigned k_Descriptor_Size = 16; + +enum AvbDescriptorTag +{ + AVB_DESCRIPTOR_TAG_PROPERTY, + AVB_DESCRIPTOR_TAG_HASHTREE, + AVB_DESCRIPTOR_TAG_HASH, + AVB_DESCRIPTOR_TAG_KERNEL_CMDLINE, + AVB_DESCRIPTOR_TAG_CHAIN_PARTITION +}; + +struct AvbDescriptor +{ + UInt64 Tag; + UInt64 Size; + + void Parse(const Byte *p) + { + G64 (0, Tag); + G64 (8, Size); + } +}; + + +enum AvbHashtreeDescriptorFlags +{ + AVB_HASHTREE_DESCRIPTOR_FLAGS_DO_NOT_USE_AB = (1 << 0), + AVB_HASHTREE_DESCRIPTOR_FLAGS_CHECK_AT_MOST_ONCE = (1 << 1) +}; + +/* A descriptor containing information about a dm-verity hashtree. + * + * Hash-trees are used to verify large partitions typically containing + * file systems. See + * https://gitlab.com/cryptsetup/cryptsetup/wikis/DMVerity for more + * information about dm-verity. + * + * Following this struct are |partition_name_len| bytes of the + * partition name (UTF-8 encoded), |salt_len| bytes of salt, and then + * |root_digest_len| bytes of the root digest. + * + * The |reserved| field is for future expansion and must be set to NUL + * bytes. + * + * Changes in v1.1: + * - flags field is added which supports AVB_HASHTREE_DESCRIPTOR_FLAGS_USE_AB + * - digest_len may be zero, which indicates the use of a persistent digest + */ + +static const unsigned k_Hashtree_Size_Min = 164; + +struct AvbHashtreeDescriptor +{ + UInt32 dm_verity_version; + UInt64 image_size; + UInt64 tree_offset; + UInt64 tree_size; + UInt32 data_block_size; + UInt32 hash_block_size; + UInt32 fec_num_roots; + UInt64 fec_offset; + UInt64 fec_size; + Byte hash_algorithm[32]; + UInt32 partition_name_len; + UInt32 salt_len; + UInt32 root_digest_len; + UInt32 flags; + Byte reserved[60]; + void Parse(const Byte *p) + { + G32 (0, dm_verity_version); + G64 (4, image_size); + G64 (12, tree_offset); + G64 (20, tree_size); + G32 (28, data_block_size); + G32 (32, hash_block_size); + G32 (36, fec_num_roots); + G64 (40, fec_offset); + G64 (48, fec_size); + memcpy(hash_algorithm, p + 56, 32); + G32 (88, partition_name_len); + G32 (92, salt_len); + G32 (96, root_digest_len); + G32 (100, flags); + } +}; + +static const unsigned k_PropertyDescriptor_Size_Min = 16; + +struct AvbPropertyDescriptor +{ + UInt64 key_num_bytes; + UInt64 value_num_bytes; + + void Parse(const Byte *p) + { + G64 (0, key_num_bytes); + G64 (8, value_num_bytes); + } +}; + +Z7_class_CHandler_final: public CHandlerCont +{ + Z7_IFACE_COM7_IMP(IInArchive_Cont) + + // UInt64 _startOffset; + UInt64 _phySize; + + CFooter Footer; + AString Name; + const char *Ext; + + HRESULT Open2(IInStream *stream); + + virtual int GetItem_ExtractInfo(UInt32 index, UInt64 &pos, UInt64 &size) const Z7_override + { + if (index != 0) + return NExtract::NOperationResult::kUnavailable; + // pos = _startOffset; + pos = 0; + size = Footer.original_image_size; + return NExtract::NOperationResult::kOK; + } +}; + + +HRESULT CHandler::Open2(IInStream *stream) +{ + UInt64 fileSize; + { + Byte buf[kFooterSize]; + RINOK(InStream_GetSize_SeekToEnd(stream, fileSize)) + if (fileSize < kFooterSize) + return S_FALSE; + RINOK(InStream_SeekSet(stream, fileSize - kFooterSize)) + RINOK(ReadStream_FALSE(stream, buf, kFooterSize)) + if (memcmp(buf, k_Signature, k_SignatureSize) != 0) + return S_FALSE; + Footer.Parse(buf); + if (Footer.vbmeta_size > VBMETA_MAX_SIZE || + Footer.vbmeta_size < AVB_VBMETA_IMAGE_HEADER_SIZE) + return S_FALSE; + for (unsigned i = 36; i < kFooterSize; i++) + if (buf[i] != 0) + return S_FALSE; + } + { + CByteBuffer buf; + buf.Alloc((size_t)Footer.vbmeta_size); + RINOK(InStream_SeekSet(stream, Footer.vbmeta_offset)) + RINOK(ReadStream_FALSE(stream, buf, (size_t)Footer.vbmeta_size)) + + AvbVBMetaImageHeader meta; + if (!meta.Parse(buf)) + return S_FALSE; + + unsigned offset = (unsigned)AVB_VBMETA_IMAGE_HEADER_SIZE; + unsigned rem = (unsigned)(Footer.vbmeta_size - offset); + + if (meta.authentication_data_block_size != 0) + { + if (rem < meta.authentication_data_block_size) + return S_FALSE; + const unsigned u = (unsigned)meta.authentication_data_block_size; + offset += u; + rem -= u; + } + + if (rem < meta.descriptors_offset || + rem - meta.descriptors_offset < meta.descriptors_size) + return S_FALSE; + rem = (unsigned)meta.descriptors_size; + while (rem != 0) + { + if (rem < k_Descriptor_Size) + return S_FALSE; + AvbDescriptor desc; + desc.Parse(buf + offset); + offset += k_Descriptor_Size; + rem -= k_Descriptor_Size; + if (desc.Size > rem) + return S_FALSE; + const unsigned descSize = (unsigned)desc.Size; + if (desc.Tag == AVB_DESCRIPTOR_TAG_HASHTREE) + { + if (descSize < k_Hashtree_Size_Min) + return S_FALSE; + AvbHashtreeDescriptor ht; + ht.Parse(buf + offset); + unsigned pos = k_Hashtree_Size_Min; + + if (pos + ht.partition_name_len > descSize) + return S_FALSE; + Name.Empty(); // UTF-8 + AddNameToString(Name, buf + offset + pos, ht.partition_name_len, false); + pos += ht.partition_name_len; + + if (pos + ht.salt_len > descSize) + return S_FALSE; + CByteBuffer salt; + salt.CopyFrom(buf + offset + pos, ht.salt_len); + pos += ht.salt_len; + + if (pos + ht.root_digest_len > descSize) + return S_FALSE; + CByteBuffer digest; + digest.CopyFrom(buf + offset + pos, ht.root_digest_len); + pos += ht.root_digest_len; + // what is that digest? + } + else if (desc.Tag == AVB_DESCRIPTOR_TAG_PROPERTY) + { + if (descSize < k_PropertyDescriptor_Size_Min + 2) + return S_FALSE; + AvbPropertyDescriptor pt; + pt.Parse(buf + offset); + unsigned pos = k_PropertyDescriptor_Size_Min; + + if (pt.key_num_bytes > descSize - pos - 1) + return S_FALSE; + AString key; // UTF-8 + AddNameToString(key, buf + offset + pos, (unsigned)pt.key_num_bytes, false); + pos += (unsigned)pt.key_num_bytes + 1; + + if (descSize < pos) + return S_FALSE; + if (pt.value_num_bytes > descSize - pos - 1) + return S_FALSE; + AString value; // UTF-8 + AddNameToString(value, buf + offset + pos, (unsigned)pt.value_num_bytes, false); + pos += (unsigned)pt.value_num_bytes + 1; + } + offset += descSize; + rem -= descSize; + } + + _phySize = fileSize; + + // _startOffset = 0; + return S_OK; + } +} + + +Z7_COM7F_IMF(CHandler::Open(IInStream *stream, + const UInt64 * /* maxCheckStartPosition */, + IArchiveOpenCallback * /* openArchiveCallback */)) +{ + COM_TRY_BEGIN + Close(); + try + { + if (Open2(stream) != S_OK) + return S_FALSE; + _stream = stream; + + { + CMyComPtr parseStream; + if (GetStream(0, &parseStream) == S_OK && parseStream) + { + const size_t kParseSize = 1 << 11; + Byte buf[kParseSize]; + if (ReadStream_FAIL(parseStream, buf, kParseSize) == S_OK) + { + UInt64 extSize; + if (NExt::IsArc_Ext_PhySize(buf, kParseSize, &extSize) == k_IsArc_Res_YES) + if (extSize == Footer.original_image_size) + Ext = "ext"; + } + } + } + } + catch(...) { return S_FALSE; } + return S_OK; + COM_TRY_END +} + + +Z7_COM7F_IMF(CHandler::Close()) +{ + _stream.Release(); + // _startOffset = 0; + _phySize = 0; + Ext = NULL; + Name.Empty(); + return S_OK; +} + + +static const Byte kArcProps[] = +{ + kpidName +}; + +static const Byte kProps[] = +{ + kpidSize, + kpidPackSize, +}; + +IMP_IInArchive_Props +IMP_IInArchive_ArcProps + + +Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) +{ + COM_TRY_BEGIN + NCOM::CPropVariant prop; + switch (propID) + { + case kpidMainSubfile: prop = (UInt32)0; break; + case kpidPhySize: prop = _phySize; break; + case kpidName: + { + if (!Name.IsEmpty()) + { + AString s (Name); + s += ".avb"; + prop = s; + } + break; + } + } + prop.Detach(value); + return S_OK; + COM_TRY_END +} + + +Z7_COM7F_IMF(CHandler::GetProperty(UInt32 /* index */, PROPID propID, PROPVARIANT *value)) +{ + COM_TRY_BEGIN + NCOM::CPropVariant prop; + + switch (propID) + { + case kpidPath: + { + if (!Name.IsEmpty()) + { + AString s (Name); + s += '.'; + s += Ext ? Ext : "img"; + prop = s; + } + break; + } + case kpidPackSize: + case kpidSize: + prop = Footer.original_image_size; + break; + case kpidExtension: prop = (Ext ? Ext : "img"); break; + } + + prop.Detach(value); + return S_OK; + COM_TRY_END +} + + +Z7_COM7F_IMF(CHandler::GetNumberOfItems(UInt32 *numItems)) +{ + *numItems = 1; + return S_OK; +} + + +REGISTER_ARC_I_NO_SIG( + "AVB", "avb img", NULL, 0xc0, + /* k_Signature, */ + 0, + /* NArcInfoFlags::kUseGlobalOffset | */ + NArcInfoFlags::kBackwardOpen + , + NULL) + +}} diff --git a/CPP/7zip/Archive/Base64Handler.cpp b/CPP/7zip/Archive/Base64Handler.cpp index 5b06051..4df491c 100644 --- a/CPP/7zip/Archive/Base64Handler.cpp +++ b/CPP/7zip/Archive/Base64Handler.cpp @@ -369,18 +369,15 @@ Z7_COM7F_IMF(CHandler::Open(IInStream *stream, const UInt64 *, IArchiveOpenCallb _data.Alloc(kStartSize); size_t size = kStartSize; RINOK(ReadStream(stream, _data, &size)) - UInt32 isArcRes = IsArc_Base64(_data, size); - if (isArcRes == k_IsArc_Res_NO) + if (IsArc_Base64(_data, size) == k_IsArc_Res_NO) return S_FALSE; } _isArc = true; UInt64 packSize64; RINOK(InStream_GetSize_SeekToEnd(stream, packSize64)) - if (packSize64 == 0) return S_FALSE; - size_t curSize = 1 << 16; if (curSize > packSize64) curSize = (size_t)packSize64; @@ -447,35 +444,24 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, return E_INVALIDARG; RINOK(extractCallback->SetTotal(_size)) - - CLocalProgress *lps = new CLocalProgress; - CMyComPtr progress = lps; + CMyComPtr2_Create lps; lps->Init(extractCallback, false); - + // RINOK(lps->SetCur()) + Int32 opRes; { - lps->InSize = lps->OutSize = 0; - RINOK(lps->SetCur()) - CMyComPtr realOutStream; const Int32 askMode = testMode ? NExtract::NAskMode::kTest : NExtract::NAskMode::kExtract; - RINOK(extractCallback->GetStream(0, &realOutStream, askMode)) - if (!testMode && !realOutStream) return S_OK; - - extractCallback->PrepareOperation(askMode); - + RINOK(extractCallback->PrepareOperation(askMode)) if (realOutStream) { RINOK(WriteStream(realOutStream, (const Byte *)_data, _size)) - realOutStream.Release(); } - - Int32 opRes = NExtract::NOperationResult::kOK; - + opRes = NExtract::NOperationResult::kOK; if (_sres != k_Base64_RES_Finished) { if (_sres == k_Base64_RES_NeedMoreInput) @@ -483,14 +469,11 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, else if (_sres == k_Base64_RES_UnexpectedChar) opRes = NExtract::NOperationResult::kDataError; } - - RINOK(extractCallback->SetOperationResult(opRes)) } - + RINOK(extractCallback->SetOperationResult(opRes)) lps->InSize = _phySize; lps->OutSize = _size; return lps->SetCur(); - COM_TRY_END } diff --git a/CPP/7zip/Archive/Bz2Handler.cpp b/CPP/7zip/Archive/Bz2Handler.cpp index 994b1ad..ffd7ad0 100644 --- a/CPP/7zip/Archive/Bz2Handler.cpp +++ b/CPP/7zip/Archive/Bz2Handler.cpp @@ -77,7 +77,9 @@ Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) if (_needMoreInput) v |= kpv_ErrorFlags_UnexpectedEnd; if (_dataAfterEnd) v |= kpv_ErrorFlags_DataAfterEnd; prop = v; + break; } + default: break; } prop.Detach(value); return S_OK; @@ -96,6 +98,7 @@ Z7_COM7F_IMF(CHandler::GetProperty(UInt32 /* index */, PROPID propID, PROPVARIAN { case kpidPackSize: if (_packSize_Defined) prop = _packSize; break; case kpidSize: if (_unpackSize_Defined) prop = _unpackSize; break; + default: break; } prop.Detach(value); return S_OK; @@ -175,8 +178,12 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, return E_INVALIDARG; if (_packSize_Defined) - extractCallback->SetTotal(_packSize); + { + RINOK(extractCallback->SetTotal(_packSize)) + } + Int32 opRes; + { CMyComPtr realOutStream; const Int32 askMode = testMode ? NExtract::NAskMode::kTest : @@ -185,7 +192,7 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, if (!testMode && !realOutStream) return S_OK; - extractCallback->PrepareOperation(askMode); + RINOK(extractCallback->PrepareOperation(askMode)) if (_needSeekToStart) { @@ -198,90 +205,85 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, // try { - NCompress::NBZip2::CDecoder *decoderSpec = new NCompress::NBZip2::CDecoder; - CMyComPtr decoder = decoderSpec; + CMyComPtr2_Create decoder; #ifndef Z7_ST - RINOK(decoderSpec->SetNumberOfThreads(_props._numThreads)) + RINOK(decoder->SetNumberOfThreads(_props._numThreads)) #endif - CDummyOutStream *outStreamSpec = new CDummyOutStream; - CMyComPtr outStream(outStreamSpec); - outStreamSpec->SetStream(realOutStream); - outStreamSpec->Init(); - - realOutStream.Release(); + CMyComPtr2_Create outStream; + outStream->SetStream(realOutStream); + outStream->Init(); + // realOutStream.Release(); - CLocalProgress *lps = new CLocalProgress; - CMyComPtr progress = lps; + CMyComPtr2_Create lps; lps->Init(extractCallback, true); - decoderSpec->FinishMode = true; - decoderSpec->Base.DecodeAllStreams = true; + decoder->FinishMode = true; + decoder->Base.DecodeAllStreams = true; _dataAfterEnd = false; _needMoreInput = false; - lps->InSize = 0; - lps->OutSize = 0; - - HRESULT result = decoder->Code(_seqStream, outStream, NULL, NULL, progress); + HRESULT result = decoder.Interface()->Code(_seqStream, outStream, NULL, NULL, lps); if (result != S_FALSE && result != S_OK) return result; - if (decoderSpec->Base.NumStreams == 0) + if (decoder->Base.NumStreams == 0) { _isArc = false; result = S_FALSE; } else { - const UInt64 inProcessedSize = decoderSpec->GetInputProcessedSize(); + const UInt64 inProcessedSize = decoder->GetInputProcessedSize(); UInt64 packSize = inProcessedSize; - if (decoderSpec->Base.NeedMoreInput) + if (decoder->Base.NeedMoreInput) _needMoreInput = true; - if (!decoderSpec->Base.IsBz) + if (!decoder->Base.IsBz) { - packSize = decoderSpec->Base.FinishedPackSize; + packSize = decoder->Base.FinishedPackSize; if (packSize != inProcessedSize) _dataAfterEnd = true; } _packSize = packSize; - _unpackSize = decoderSpec->GetOutProcessedSize(); - _numStreams = decoderSpec->Base.NumStreams; - _numBlocks = decoderSpec->GetNumBlocks(); + _unpackSize = decoder->GetOutProcessedSize(); + _numStreams = decoder->Base.NumStreams; + _numBlocks = decoder->GetNumBlocks(); _packSize_Defined = true; _unpackSize_Defined = true; _numStreams_Defined = true; _numBlocks_Defined = true; + + // RINOK( + lps.Interface()->SetRatioInfo(&packSize, &_unpackSize); } - outStream.Release(); - - Int32 opRes; + // outStream.Release(); if (!_isArc) opRes = NExtract::NOperationResult::kIsNotArc; else if (_needMoreInput) opRes = NExtract::NOperationResult::kUnexpectedEnd; - else if (decoderSpec->GetCrcError()) + else if (decoder->GetCrcError()) opRes = NExtract::NOperationResult::kCRCError; else if (_dataAfterEnd) opRes = NExtract::NOperationResult::kDataAfterEnd; else if (result == S_FALSE) opRes = NExtract::NOperationResult::kDataError; - else if (decoderSpec->Base.MinorError) + else if (decoder->Base.MinorError) opRes = NExtract::NOperationResult::kDataError; else if (result == S_OK) opRes = NExtract::NOperationResult::kOK; else return result; + } return extractCallback->SetOperationResult(opRes); // } catch(...) { return E_FAIL; } @@ -352,14 +354,13 @@ static HRESULT UpdateArchive( } } RINOK(updateCallback->SetTotal(unpackSize)) - CLocalProgress *localProgressSpec = new CLocalProgress; - CMyComPtr localProgress = localProgressSpec; - localProgressSpec->Init(updateCallback, true); + + CMyComPtr2_Create lps; + lps->Init(updateCallback, true); { - NCompress::NBZip2::CEncoder *encoderSpec = new NCompress::NBZip2::CEncoder; - CMyComPtr encoder = encoderSpec; - RINOK(props.SetCoderProps(encoderSpec, NULL)) - RINOK(encoder->Code(fileInStream, outStream, NULL, NULL, localProgress)) + CMyComPtr2_Create encoder; + RINOK(props.SetCoderProps(encoder.ClsPtr(), NULL)) + RINOK(encoder.Interface()->Code(fileInStream, outStream, NULL, NULL, lps)) /* if (reportArcProp) { @@ -436,8 +437,7 @@ Z7_COM7F_IMF(CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt if (indexInArchive != 0) return E_INVALIDARG; - CLocalProgress *lps = new CLocalProgress; - CMyComPtr progress = lps; + CMyComPtr2_Create lps; lps->Init(updateCallback, true); Z7_DECL_CMyComPtr_QI_FROM( @@ -453,7 +453,7 @@ Z7_COM7F_IMF(CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt if (_stream) RINOK(InStream_SeekToBegin(_stream)) - return NCompress::CopyStream(_stream, outStream, progress); + return NCompress::CopyStream(_stream, outStream, lps); // return ReportArcProps(reportArcProp, NULL, NULL); diff --git a/CPP/7zip/Archive/Cab/CabBlockInStream.cpp b/CPP/7zip/Archive/Cab/CabBlockInStream.cpp index dc767a4..5e1fe8b 100644 --- a/CPP/7zip/Archive/Cab/CabBlockInStream.cpp +++ b/CPP/7zip/Archive/Cab/CabBlockInStream.cpp @@ -12,89 +12,111 @@ namespace NArchive { namespace NCab { -static const UInt32 kBlockSize = (1 << 16); +static const UInt32 kBlockSize = 1 << 16; +static const unsigned k_OverReadPadZone_Size = 32; +static const unsigned kHeaderSize = 8; +static const unsigned kReservedMax = 256; +static const unsigned kHeaderOffset = kBlockSize + k_OverReadPadZone_Size; -bool CCabBlockInStream::Create() +bool CBlockPackData::Create() throw() { if (!_buf) - _buf = (Byte *)::MyAlloc(kBlockSize); + _buf = (Byte *)z7_AlignedAlloc(kBlockSize + k_OverReadPadZone_Size + kHeaderSize + kReservedMax); return _buf != NULL; } -CCabBlockInStream::~CCabBlockInStream() +CBlockPackData::~CBlockPackData() throw() { - ::MyFree(_buf); + z7_AlignedFree(_buf); } -static UInt32 CheckSum(const Byte *p, UInt32 size) +static UInt32 CheckSum(const Byte *p, UInt32 size) throw() { +#ifdef MY_CPU_64BIT + + UInt64 sum64 = 0; + if (size >= 16) + { + const Byte *lim = p + (size_t)size - 16; + do + { + sum64 ^= GetUi64(p) ^ GetUi64(p + 8); + p += 16; + } + while (p <= lim); + size = (UInt32)(lim + 16 - p); + } + if (size >= 8) + { + sum64 ^= GetUi64(p); + p += 8; + size -= 8; + } + + UInt32 sum = (UInt32)(sum64 >> 32) ^ (UInt32)sum64; + +#else + UInt32 sum = 0; - - for (; size >= 8; size -= 8) + if (size >= 16) + { + const Byte *lim = p + (size_t)size - 16; + do + { + sum ^= GetUi32(p) + ^ GetUi32(p + 4) + ^ GetUi32(p + 8) + ^ GetUi32(p + 12); + p += 16; + } + while (p <= lim); + size = (UInt32)(lim + 16 - p); + } + if (size >= 8) { - sum ^= GetUi32(p) ^ GetUi32(p + 4); + sum ^= GetUi32(p + 0) ^ GetUi32(p + 4); p += 8; + size -= 8; } + +#endif if (size >= 4) { sum ^= GetUi32(p); p += 4; } - - size &= 3; - if (size > 2) sum ^= (UInt32)(*p++) << 16; - if (size > 1) sum ^= (UInt32)(*p++) << 8; - if (size > 0) sum ^= (UInt32)(*p++); - + if (size &= 3) + { + if (size >= 2) + { + if (size > 2) + sum ^= (UInt32)(*p++) << 16; + sum ^= (UInt32)(*p++) << 8; + } + sum ^= (UInt32)(*p++); + } return sum; } -HRESULT CCabBlockInStream::PreRead(ISequentialInStream *stream, UInt32 &packSize, UInt32 &unpackSize) + +HRESULT CBlockPackData::Read(ISequentialInStream *stream, Byte ReservedSize, UInt32 &packSizeRes, UInt32 &unpackSize) throw() { - const UInt32 kHeaderSize = 8; - const UInt32 kReservedMax = 256; - Byte header[kHeaderSize + kReservedMax]; - RINOK(ReadStream_FALSE(stream, header, kHeaderSize + ReservedSize)) - packSize = GetUi16(header + 4); - unpackSize = GetUi16(header + 6); + const UInt32 reserved8 = kHeaderSize + ReservedSize; + const Byte *header = _buf + kHeaderOffset; + RINOK(ReadStream_FALSE(stream, (void *)header, reserved8)) + unpackSize = GetUi16a(header + 6); + const UInt32 packSize = GetUi16a(header + 4); + packSizeRes = packSize; if (packSize > kBlockSize - _size) return S_FALSE; RINOK(ReadStream_FALSE(stream, _buf + _size, packSize)) - - if (MsZip) - { - if (_size == 0) - { - if (packSize < 2 || _buf[0] != 0x43 || _buf[1] != 0x4B) - return S_FALSE; - _pos = 2; - } - if (_size + packSize > ((UInt32)1 << 15) + 12) /* v9.31 fix. MSZIP specification */ - return S_FALSE; - } - - if (GetUi32(header) != 0) // checkSum - if (CheckSum(header, kHeaderSize + ReservedSize) != CheckSum(_buf + _size, packSize)) + memset(_buf + _size + packSize, 0xff, k_OverReadPadZone_Size); + if (*(const UInt32 *)(const void *)header != 0) // checkSum + if (CheckSum(header, reserved8) != CheckSum(_buf + _size, packSize)) return S_FALSE; - _size += packSize; return S_OK; } -Z7_COM7F_IMF(CCabBlockInStream::Read(void *data, UInt32 size, UInt32 *processedSize)) -{ - if (size != 0) - { - UInt32 rem = _size - _pos; - if (size > rem) - size = rem; - memcpy(data, _buf + _pos, size); - _pos += size; - } - if (processedSize) - *processedSize = size; - return S_OK; -} - }} diff --git a/CPP/7zip/Archive/Cab/CabBlockInStream.h b/CPP/7zip/Archive/Cab/CabBlockInStream.h index d14fff8..7568779 100644 --- a/CPP/7zip/Archive/Cab/CabBlockInStream.h +++ b/CPP/7zip/Archive/Cab/CabBlockInStream.h @@ -3,35 +3,24 @@ #ifndef ZIP7_INC_CAB_BLOCK_IN_STREAM_H #define ZIP7_INC_CAB_BLOCK_IN_STREAM_H -#include "../../../Common/MyCom.h" #include "../../IStream.h" namespace NArchive { namespace NCab { -Z7_CLASS_IMP_NOQIB_1( - CCabBlockInStream - , ISequentialInStream -) +class CBlockPackData +{ Byte *_buf; UInt32 _size; - UInt32 _pos; - public: - UInt32 ReservedSize; // < 256 - bool MsZip; - - CCabBlockInStream(): _buf(NULL), ReservedSize(0), MsZip(false) {} - ~CCabBlockInStream(); - - bool Create(); - - void InitForNewBlock() { _size = 0; _pos = 0; } - - HRESULT PreRead(ISequentialInStream *stream, UInt32 &packSize, UInt32 &unpackSize); - - UInt32 GetPackSizeAvail() const { return _size - _pos; } - const Byte *GetData() const { return _buf + _pos; } + CBlockPackData(): _buf(NULL), _size(0) {} + ~CBlockPackData() throw(); + bool Create() throw(); + void InitForNewBlock() { _size = 0; } + HRESULT Read(ISequentialInStream *stream, Byte ReservedSize, UInt32 &packSize, UInt32 &unpackSize) throw(); + UInt32 GetPackSize() const { return _size; } + // 32 bytes of overread zone is available after PackSize: + const Byte *GetData() const { return _buf; } }; }} diff --git a/CPP/7zip/Archive/Cab/CabHandler.cpp b/CPP/7zip/Archive/Cab/CabHandler.cpp index f015145..0ad2a0e 100644 --- a/CPP/7zip/Archive/Cab/CabHandler.cpp +++ b/CPP/7zip/Archive/Cab/CabHandler.cpp @@ -5,7 +5,9 @@ // #include #include "../../../../C/Alloc.h" +#include "../../../../C/CpuArch.h" +#include "../../../Common/AutoPtr.h" #include "../../../Common/ComTry.h" #include "../../../Common/IntToString.h" #include "../../../Common/StringConvert.h" @@ -15,9 +17,9 @@ #include "../../../Windows/TimeUtils.h" #include "../../Common/ProgressUtils.h" +#include "../../Common/StreamObjects.h" #include "../../Common/StreamUtils.h" -#include "../../Compress/CopyCoder.h" #include "../../Compress/DeflateDecoder.h" #include "../../Compress/LzxDecoder.h" #include "../../Compress/QuantumDecoder.h" @@ -112,14 +114,14 @@ Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) FOR_VECTOR (i, folders) { const CFolder &folder = folders[i]; - unsigned method = folder.GetMethod(); + const unsigned method = folder.GetMethod(); mask |= ((UInt32)1 << method); if (method == NHeader::NMethod::kLZX || method == NHeader::NMethod::kQuantum) { - unsigned di = (method == NHeader::NMethod::kQuantum) ? 0 : 1; + const unsigned di = (method == NHeader::NMethod::kQuantum) ? 0 : 1; if (params[di] < folder.MethodMinor) - params[di] = folder.MethodMinor; + params[di] = folder.MethodMinor; } } } @@ -227,7 +229,7 @@ Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) { AString s; s.Add_UInt32(ai.SetID); - s += '_'; + s.Add_Char('_'); s.Add_UInt32(ai.CabinetNumber + 1); s += ".cab"; prop = s; @@ -261,6 +263,7 @@ Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) } // case kpidShortComment: + default: break; } prop.Detach(value); return S_OK; @@ -274,7 +277,7 @@ Z7_COM7F_IMF(CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val const CMvItem &mvItem = m_Database.Items[index]; const CDatabaseEx &db = m_Database.Volumes[mvItem.VolumeIndex]; - unsigned itemIndex = mvItem.ItemIndex; + const unsigned itemIndex = mvItem.ItemIndex; const CItem &item = db.Items[itemIndex]; switch (propID) { @@ -321,6 +324,8 @@ Z7_COM7F_IMF(CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val case kpidVolume: prop = (UInt32)mvItem.VolumeIndex; break; #endif + + default: break; } prop.Detach(value); return S_OK; @@ -361,7 +366,7 @@ Z7_COM7F_IMF(CHandler::Open(IInStream *inStream, if (res == S_OK && !m_Database.Volumes.IsEmpty()) { const CArchInfo &lastArc = m_Database.Volumes.Back().ArcInfo; - unsigned cabNumber = db.ArcInfo.CabinetNumber; + const unsigned cabNumber = db.ArcInfo.CabinetNumber; if (lastArc.SetID != db.ArcInfo.SetID) res = S_FALSE; else if (prevChecked) @@ -479,7 +484,7 @@ Z7_COM7F_IMF(CHandler::Open(IInStream *inStream, break; } - HRESULT result = openVolumeCallback->GetStream(fullName, &nextStream); + const HRESULT result = openVolumeCallback->GetStream(fullName, &nextStream); if (result == S_OK) break; if (result != S_FALSE) @@ -539,6 +544,11 @@ Z7_CLASS_IMP_NOQIB_1( CFolderOutStream , ISequentialOutStream ) + bool m_TestMode; + bool TempBufMode; + bool m_IsOk; + bool m_FileIsOpen; + const CMvDatabaseEx *m_Database; const CRecordVector *m_ExtractStatuses; @@ -546,21 +556,18 @@ Z7_CLASS_IMP_NOQIB_1( UInt32 TempBufSize; UInt32 TempBufWritten; unsigned NumIdenticalFiles; - bool TempBufMode; unsigned m_StartIndex; unsigned m_CurrentIndex; - CMyComPtr m_ExtractCallback; - bool m_TestMode; - CMyComPtr m_RealOutStream; - - bool m_IsOk; - bool m_FileIsOpen; UInt32 m_RemainFileSize; + UInt64 m_FolderSize; UInt64 m_PosInFolder; + CMyComPtr m_ExtractCallback; + CMyComPtr m_RealOutStream; + void FreeTempBuf() { ::MyFree(TempBuf); @@ -707,10 +714,10 @@ HRESULT CFolderOutStream::WriteEmptyFiles() { const CMvItem &mvItem = m_Database->Items[m_StartIndex + m_CurrentIndex]; const CItem &item = m_Database->Volumes[mvItem.VolumeIndex].Items[mvItem.ItemIndex]; - UInt64 fileSize = item.Size; + const UInt64 fileSize = item.Size; if (fileSize != 0) return S_OK; - HRESULT result = OpenFile(); + const HRESULT result = OpenFile(); m_RealOutStream.Release(); RINOK(result) RINOK(m_ExtractCallback->SetOperationResult(NExtract::NOperationResult::kOK)) @@ -819,14 +826,14 @@ Z7_COM7F_IMF(CFolderOutStream::Write(const void *data, UInt32 size, UInt32 *proc m_RemainFileSize = item.Size; - UInt32 fileOffset = item.Offset; + const UInt32 fileOffset = item.Offset; if (fileOffset < m_PosInFolder) return E_FAIL; if (fileOffset > m_PosInFolder) { - UInt32 numBytesToWrite = MyMin(fileOffset - (UInt32)m_PosInFolder, size); + const UInt32 numBytesToWrite = MyMin(fileOffset - (UInt32)m_PosInFolder, size); realProcessed += numBytesToWrite; if (processedSize) *processedSize = realProcessed; @@ -869,7 +876,7 @@ HRESULT CFolderOutStream::FlushCorrupted(unsigned folderIndex) { if (!NeedMoreWrite()) return S_OK; - UInt64 remain = GetRemain(); + const UInt64 remain = GetRemain(); UInt32 size = (UInt32)1 << 20; if (size > remain) size = (UInt32)remain; @@ -883,7 +890,7 @@ HRESULT CFolderOutStream::Unsupported() { while (m_CurrentIndex < m_ExtractStatuses->Size()) { - HRESULT result = OpenFile(); + const HRESULT result = OpenFile(); if (result != S_FALSE && result != S_OK) return result; m_RealOutStream.Release(); @@ -895,7 +902,7 @@ HRESULT CFolderOutStream::Unsupported() Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, - Int32 testModeSpec, IArchiveExtractCallback *extractCallback)) + Int32 testMode, IArchiveExtractCallback *extractCallback)) { COM_TRY_BEGIN const bool allFilesMode = (numItems == (UInt32)(Int32)-1); @@ -903,7 +910,6 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, numItems = m_Database.Items.Size(); if (numItems == 0) return S_OK; - bool testMode = (testModeSpec != 0); UInt64 totalUnPacked = 0; UInt32 i; @@ -912,12 +918,12 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, for (i = 0; i < numItems; i++) { - unsigned index = allFilesMode ? i : indices[i]; + const unsigned index = allFilesMode ? i : indices[i]; const CMvItem &mvItem = m_Database.Items[index]; const CItem &item = m_Database.Volumes[mvItem.VolumeIndex].Items[mvItem.ItemIndex]; if (item.IsDir()) continue; - int folderIndex = m_Database.GetFolderIndex(&mvItem); + const int folderIndex = m_Database.GetFolderIndex(&mvItem); if (folderIndex != lastFolder) totalUnPacked += lastFolderSize; lastFolder = folderIndex; @@ -925,42 +931,31 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, } totalUnPacked += lastFolderSize; + RINOK(extractCallback->SetTotal(totalUnPacked)) - extractCallback->SetTotal(totalUnPacked); - - totalUnPacked = 0; - - UInt64 totalPacked = 0; - - CLocalProgress *lps = new CLocalProgress; - CMyComPtr progress = lps; + CMyComPtr2_Create lps; lps->Init(extractCallback, false); - NCompress::CCopyCoder *copyCoderSpec = new NCompress::CCopyCoder; - CMyComPtr copyCoder = copyCoderSpec; - - NCompress::NDeflate::NDecoder::CCOMCoder *deflateDecoderSpec = NULL; - CMyComPtr deflateDecoder; - - NCompress::NLzx::CDecoder *lzxDecoderSpec = NULL; - CMyComPtr lzxDecoder; - - NCompress::NQuantum::CDecoder *quantumDecoderSpec = NULL; - CMyComPtr quantumDecoder; + CMyComPtr2 deflateDecoder; + CMyUniquePtr lzxDecoder; + CMyUniquePtr quantumDecoder; - CCabBlockInStream *cabBlockInStreamSpec = new CCabBlockInStream(); - CMyComPtr cabBlockInStream = cabBlockInStreamSpec; - if (!cabBlockInStreamSpec->Create()) + CBlockPackData blockPackData; + if (!blockPackData.Create()) return E_OUTOFMEMORY; + CMyComPtr2_Create inBufStream; + CRecordVector extractStatuses; + + totalUnPacked = 0; + UInt64 totalPacked = 0; for (i = 0;;) { lps->OutSize = totalUnPacked; lps->InSize = totalPacked; RINOK(lps->SetCur()) - if (i >= numItems) break; @@ -1028,8 +1023,7 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, curUnpack = item2.GetEndOffset(); } - CFolderOutStream *cabFolderOutStream = new CFolderOutStream; - CMyComPtr outStream(cabFolderOutStream); + CMyComPtr2_Create cabFolderOutStream; const int folderIndex2 = item.GetFolderIndex(db.Folders.Size()); if (folderIndex2 < 0) @@ -1037,9 +1031,8 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, const CFolder &folder = db.Folders[(unsigned)folderIndex2]; cabFolderOutStream->Init(&m_Database, &extractStatuses, startIndex2, - curUnpack, extractCallback, testMode); + curUnpack, extractCallback, testMode != 0); - cabBlockInStreamSpec->MsZip = false; HRESULT res = S_OK; switch (folder.GetMethod()) @@ -1048,30 +1041,17 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, break; case NHeader::NMethod::kMSZip: - if (!deflateDecoder) - { - deflateDecoderSpec = new NCompress::NDeflate::NDecoder::CCOMCoder; - deflateDecoder = deflateDecoderSpec; - } - cabBlockInStreamSpec->MsZip = true; + deflateDecoder.Create_if_Empty(); break; case NHeader::NMethod::kLZX: - if (!lzxDecoder) - { - lzxDecoderSpec = new NCompress::NLzx::CDecoder; - lzxDecoder = lzxDecoderSpec; - } - res = lzxDecoderSpec->SetParams_and_Alloc(folder.MethodMinor); + lzxDecoder.Create_if_Empty(); + res = lzxDecoder->Set_DictBits_and_Alloc(folder.MethodMinor); break; case NHeader::NMethod::kQuantum: - if (!quantumDecoder) - { - quantumDecoderSpec = new NCompress::NQuantum::CDecoder; - quantumDecoder = quantumDecoderSpec; - } - res = quantumDecoderSpec->SetParams(folder.MethodMinor); + quantumDecoder.Create_if_Empty(); + res = quantumDecoder->SetParams(folder.MethodMinor); break; default: @@ -1109,7 +1089,6 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, if (bl == 0) { - cabBlockInStreamSpec->ReservedSize = db2.ArcInfo.GetDataBlockReserveSize(); RINOK(InStream_SeekSet(db2.Stream, db2.StartPosition + folder2.DataStart)) } @@ -1135,10 +1114,10 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, bl++; if (!keepInputBuffer) - cabBlockInStreamSpec->InitForNewBlock(); + blockPackData.InitForNewBlock(); UInt32 packSize, unpackSize; - res = cabBlockInStreamSpec->PreRead(db2.Stream, packSize, unpackSize); + res = blockPackData.Read(db2.Stream, db2.ArcInfo.GetDataBlockReserveSize(), packSize, unpackSize); if (res == S_FALSE) break; RINOK(res) @@ -1146,30 +1125,22 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, if (keepInputBuffer) continue; - UInt64 totalUnPacked2 = totalUnPacked + cabFolderOutStream->GetPosInFolder(); + const UInt64 totalUnPacked2 = totalUnPacked + cabFolderOutStream->GetPosInFolder(); totalPacked += packSize; - lps->OutSize = totalUnPacked2; - lps->InSize = totalPacked; - RINOK(lps->SetCur()) - - const UInt32 kBlockSizeMax = (1 << 15); - - /* We don't try to reduce last block. - Note that LZX converts data with x86 filter. - and filter needs larger input data than reduced size. - It's simpler to decompress full chunk here. - also we need full block for quantum for more integrity checks */ - - if (unpackSize > kBlockSizeMax) + if (totalUnPacked2 - lps->OutSize >= (1 << 26) + || totalPacked - lps->InSize >= (1 << 24)) { - res = S_FALSE; - break; + lps->OutSize = totalUnPacked2; + lps->InSize = totalPacked; + RINOK(lps->SetCur()) } + const unsigned kBlockSizeMax = 1u << 15; + if (unpackSize != kBlockSizeMax) { - if (thereWasNotAlignedChunk) + if (unpackSize > kBlockSizeMax || thereWasNotAlignedChunk) { res = S_FALSE; break; @@ -1177,55 +1148,95 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, thereWasNotAlignedChunk = true; } - UInt64 unpackSize64 = unpackSize; - UInt32 packSizeChunk = cabBlockInStreamSpec->GetPackSizeAvail(); + /* We don't try to reduce last block. + Note that LZX converts data with x86 filter. + and filter needs larger input data than reduced size. + It's simpler to decompress full chunk here. + also we need full block for quantum for more integrity checks */ + + const UInt64 unpackSize64 = unpackSize; + const UInt32 packSizeChunk = blockPackData.GetPackSize(); switch (folder2.GetMethod()) { case NHeader::NMethod::kNone: - res = copyCoder->Code(cabBlockInStream, outStream, NULL, &unpackSize64, NULL); + if (unpackSize != packSizeChunk) + { + res = S_FALSE; + break; + } + res = WriteStream(cabFolderOutStream, blockPackData.GetData(), packSizeChunk); break; case NHeader::NMethod::kMSZip: - deflateDecoderSpec->Set_KeepHistory(keepHistory); - /* v9.31: now we follow MSZIP specification that requires to finish deflate stream at the end of each block. - But PyCabArc can create CAB archives that doesn't have finish marker at the end of block. + { + /* v24.00 : fixed : we check 2-bytes MSZIP signature only + when block was constructed from all volumes. */ + const Byte *packData = blockPackData.GetData(); + if (unpackSize > (1u << 15) + 12 /* MSZIP specification */ + || packSizeChunk < 2 || GetUi16(packData) != 0x4b43) + { + res = S_FALSE; + break; + } + const UInt32 packSizeChunk_2 = packSizeChunk - 2; + inBufStream->Init(packData + 2, packSizeChunk_2); + + deflateDecoder->Set_KeepHistory(keepHistory); + /* v9.31: now we follow MSZIP specification that requires + to finish deflate stream at the end of each block. + But PyCabArc can create CAB archives that don't have + finish marker at the end of block. Cabarc probably ignores such errors in cab archives. - Maybe we also should ignore that error? + Maybe we also should ignore such error? Or we should extract full file and show the warning? */ - deflateDecoderSpec->Set_NeedFinishInput(true); - res = deflateDecoder->Code(cabBlockInStream, outStream, NULL, &unpackSize64, NULL); + deflateDecoder->Set_NeedFinishInput(true); + res = deflateDecoder.Interface()->Code(inBufStream, cabFolderOutStream, NULL, &unpackSize64, NULL); if (res == S_OK) { - if (!deflateDecoderSpec->IsFinished()) + if (!deflateDecoder->IsFinished()) + res = S_FALSE; + if (!deflateDecoder->IsFinalBlock()) res = S_FALSE; - if (!deflateDecoderSpec->IsFinalBlock()) + if (deflateDecoder->GetInputProcessedSize() != packSizeChunk_2) res = S_FALSE; } break; + } case NHeader::NMethod::kLZX: - lzxDecoderSpec->SetKeepHistory(keepHistory); - lzxDecoderSpec->KeepHistoryForNext = true; - - res = lzxDecoderSpec->Code(cabBlockInStreamSpec->GetData(), packSizeChunk, unpackSize); - + lzxDecoder->Set_KeepHistory(keepHistory); + lzxDecoder->Set_KeepHistoryForNext(true); + res = lzxDecoder->Code_WithExceedReadWrite(blockPackData.GetData(), + packSizeChunk, unpackSize); if (res == S_OK) - res = WriteStream(outStream, - lzxDecoderSpec->GetUnpackData(), - lzxDecoderSpec->GetUnpackSize()); + res = WriteStream(cabFolderOutStream, + lzxDecoder->GetUnpackData(), + lzxDecoder->GetUnpackSize()); break; case NHeader::NMethod::kQuantum: - res = quantumDecoderSpec->Code(cabBlockInStreamSpec->GetData(), - packSizeChunk, outStream, unpackSize, keepHistory); + { + res = quantumDecoder->Code(blockPackData.GetData(), + packSizeChunk, unpackSize, keepHistory); + if (res == S_OK) + { + const UInt32 num = unpackSize; + res = WriteStream(cabFolderOutStream, + quantumDecoder->GetDataPtr() - num, num); + } + break; + } + default: + // it's unexpected case, because we checked method before + // res = E_NOTIMPL; break; } if (res != S_OK) { if (res != S_FALSE) - RINOK(res) + return res; break; } diff --git a/CPP/7zip/Archive/Cab/CabIn.cpp b/CPP/7zip/Archive/Cab/CabIn.cpp index fb69643..bd6969a 100644 --- a/CPP/7zip/Archive/Cab/CabIn.cpp +++ b/CPP/7zip/Archive/Cab/CabIn.cpp @@ -80,11 +80,10 @@ struct CSignatureFinder UInt32 _alignSize; UInt32 _bufUseCapacity; + const UInt64 *SearchLimit; ISequentialInStream *Stream; UInt64 Processed; // Global offset of start of Buf - const UInt64 *SearchLimit; - UInt32 GetTotalCapacity(UInt32 basicSize, UInt32 headerSize) { _headerSize = headerSize; @@ -111,7 +110,7 @@ HRESULT CSignatureFinder::Find() while (End - Pos >= _headerSize) { const Byte *p = Buf + Pos; - Byte b = Signature[0]; + const Byte b = Signature[0]; for (;;) { if (*p == b) { break; } p++; @@ -132,7 +131,7 @@ HRESULT CSignatureFinder::Find() if (Pos >= _alignSize) { - UInt32 num = (Pos & ~(_alignSize - 1)); + const UInt32 num = (Pos & ~(_alignSize - 1)); Processed += num; Pos -= num; End -= num; @@ -143,7 +142,7 @@ HRESULT CSignatureFinder::Find() { if (Processed + Pos > *SearchLimit) return S_FALSE; - UInt64 rem2 = *SearchLimit - (Processed + End) + _headerSize; + const UInt64 rem2 = *SearchLimit - (Processed + End) + _headerSize; if (rem > rem2) rem = (UInt32)rem2; } @@ -248,7 +247,7 @@ HRESULT CInArchive::Open2(CDatabaseEx &db, const UInt64 *searchHeaderSizeLimit) limitedStreamSpec = new CLimitedSequentialInStream; limitedStreamSpec->SetStream(db.Stream); limitedStream = limitedStreamSpec; - UInt32 remInFinder = finder.End - finder.Pos; + const UInt32 remInFinder = finder.End - finder.Pos; if (ai.Size <= remInFinder) { limitedStreamSpec->Init(0); @@ -273,7 +272,7 @@ HRESULT CInArchive::Open2(CDatabaseEx &db, const UInt64 *searchHeaderSizeLimit) _tempBuf.Alloc(1 << 12); Byte p[16]; - unsigned nextSize = 4 + (ai.ReserveBlockPresent() ? 4 : 0); + const unsigned nextSize = 4 + (ai.ReserveBlockPresent() ? 4 : 0); Read(p, nextSize); ai.SetID = Get16(p); ai.CabinetNumber = Get16(p + 2); @@ -325,8 +324,8 @@ HRESULT CInArchive::Open2(CDatabaseEx &db, const UInt64 *searchHeaderSizeLimit) item.Size = Get32(p); item.Offset = Get32(p + 4); item.FolderIndex = Get16(p + 8); - UInt16 pureDate = Get16(p + 10); - UInt16 pureTime = Get16(p + 12); + const UInt16 pureDate = Get16(p + 10); + const UInt16 pureTime = Get16(p + 12); item.Time = (((UInt32)pureDate << 16)) | pureTime; item.Attributes = Get16(p + 14); @@ -366,12 +365,12 @@ static int CompareMvItems(const CMvItem *p1, const CMvItem *p2, void *param) const CDatabaseEx &db2 = mvDb.Volumes[p2->VolumeIndex]; const CItem &item1 = db1.Items[p1->ItemIndex]; const CItem &item2 = db2.Items[p2->ItemIndex]; - bool isDir1 = item1.IsDir(); - bool isDir2 = item2.IsDir(); + const bool isDir1 = item1.IsDir(); + const bool isDir2 = item2.IsDir(); if (isDir1 && !isDir2) return -1; if (isDir2 && !isDir1) return 1; - int f1 = mvDb.GetFolderIndex(p1); - int f2 = mvDb.GetFolderIndex(p2); + const int f1 = mvDb.GetFolderIndex(p1); + const int f2 = mvDb.GetFolderIndex(p2); RINOZ(MyCompare(f1, f2)) RINOZ(MyCompare(item1.Offset, item2.Offset)) RINOZ(MyCompare(item1.Size, item2.Size)) @@ -434,7 +433,7 @@ void CMvDatabaseEx::FillSortAndShrink() FOR_VECTOR (i, Items) { - int folderIndex = GetFolderIndex(&Items[i]); + const int folderIndex = GetFolderIndex(&Items[i]); while (folderIndex >= (int)FolderStartFileIndex.Size()) FolderStartFileIndex.Add(i); } @@ -452,7 +451,7 @@ bool CMvDatabaseEx::Check() if (db0.Folders.IsEmpty() || db1.Folders.IsEmpty()) return false; const CFolder &f0 = db0.Folders.Back(); - const CFolder &f1 = db1.Folders.Front(); + const CFolder &f1 = db1.Folders.FrontItem(); if (f0.MethodMajor != f1.MethodMajor || f0.MethodMinor != f1.MethodMinor) return false; @@ -466,14 +465,14 @@ bool CMvDatabaseEx::Check() FOR_VECTOR (i, Items) { const CMvItem &mvItem = Items[i]; - int fIndex = GetFolderIndex(&mvItem); + const int fIndex = GetFolderIndex(&mvItem); if (fIndex >= (int)FolderStartFileIndex.Size()) return false; const CItem &item = Volumes[mvItem.VolumeIndex].Items[mvItem.ItemIndex]; if (item.IsDir()) continue; - int folderIndex = GetFolderIndex(&mvItem); + const int folderIndex = GetFolderIndex(&mvItem); if (folderIndex != prevFolder) prevFolder = folderIndex; diff --git a/CPP/7zip/Archive/Chm/ChmHandler.cpp b/CPP/7zip/Archive/Chm/ChmHandler.cpp index 38e2543..93c3a85 100644 --- a/CPP/7zip/Archive/Chm/ChmHandler.cpp +++ b/CPP/7zip/Archive/Chm/ChmHandler.cpp @@ -2,6 +2,7 @@ #include "StdAfx.h" +#include "../../../Common/AutoPtr.h" #include "../../../Common/ComTry.h" #include "../../../Common/StringConvert.h" #include "../../../Common/UTFConvert.h" @@ -414,21 +415,18 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, m_Database.Indices.Size()); if (numItems == 0) return S_OK; - bool testMode = (testModeSpec != 0); + const bool testMode = (testModeSpec != 0); UInt64 currentTotalSize = 0; - NCompress::CCopyCoder *copyCoderSpec = new NCompress::CCopyCoder; - CMyComPtr copyCoder = copyCoderSpec; - UInt32 i; - - CLocalProgress *lps = new CLocalProgress; - CMyComPtr progress = lps; + CMyComPtr2_Create lps; lps->Init(extractCallback, false); - CLimitedSequentialInStream *streamSpec = new CLimitedSequentialInStream; - CMyComPtr inStream(streamSpec); - streamSpec->SetStream(m_Stream); + CMyComPtr2_Create copyCoder; + CMyComPtr2_Create inStream; + inStream->SetStream(m_Stream); + + UInt32 i; if (m_Database.LowLevel) { @@ -441,7 +439,7 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, for (i = 0; i < numItems; i++) totalSize += m_Database.Items[allFilesMode ? i : indices[i]].Size; - extractCallback->SetTotal(totalSize); + RINOK(extractCallback->SetTotal(totalSize)) for (i = 0; i < numItems; i++, currentTotalSize += currentItemSize) { @@ -465,7 +463,7 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, continue; if (!testMode) { - UInt32 size = m_Database.NewFormatString.Len(); + const unsigned size = m_Database.NewFormatString.Len(); RINOK(WriteStream(realOutStream, (const char *)m_Database.NewFormatString, size)) } RINOK(extractCallback->SetOperationResult(NExtract::NOperationResult::kOK)) @@ -492,18 +490,18 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, } RINOK(InStream_SeekSet(m_Stream, m_Database.ContentOffset + item.Offset)) - streamSpec->Init(item.Size); + inStream->Init(item.Size); - RINOK(copyCoder->Code(inStream, realOutStream, NULL, NULL, progress)) + RINOK(copyCoder.Interface()->Code(inStream, realOutStream, NULL, NULL, lps)) realOutStream.Release(); - RINOK(extractCallback->SetOperationResult((copyCoderSpec->TotalSize == item.Size) ? + RINOK(extractCallback->SetOperationResult((copyCoder->TotalSize == item.Size) ? NExtract::NOperationResult::kOK: NExtract::NOperationResult::kDataError)) } return S_OK; } - UInt64 lastFolderIndex = ((UInt64)0 - 1); + UInt64 lastFolderIndex = (UInt64)0 - 1; for (i = 0; i < numItems; i++) { @@ -536,10 +534,8 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, RINOK(extractCallback->SetTotal(currentTotalSize)) - NCompress::NLzx::CDecoder *lzxDecoderSpec = NULL; - CMyComPtr lzxDecoder; - CChmFolderOutStream *chmFolderOutStream = NULL; - CMyComPtr outStream; + CMyUniquePtr lzxDecoder; + CMyComPtr2_Create chmFolderOutStream; currentTotalSize = 0; @@ -558,7 +554,7 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, i++; const CItem &item = m_Database.Items[m_Database.Indices[index]]; const UInt64 sectionIndex = item.Section; - Int32 askMode= testMode ? + const Int32 askMode= testMode ? NExtract::NAskMode::kTest : NExtract::NAskMode::kExtract; @@ -586,9 +582,9 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, if (!testMode && item.Size != 0) { RINOK(InStream_SeekSet(m_Stream, m_Database.ContentOffset + item.Offset)) - streamSpec->Init(item.Size); - RINOK(copyCoder->Code(inStream, realOutStream, NULL, NULL, progress)) - if (copyCoderSpec->TotalSize != item.Size) + inStream->Init(item.Size); + RINOK(copyCoder.Interface()->Code(inStream, realOutStream, NULL, NULL, lps)) + if (copyCoder->TotalSize != item.Size) opRes = NExtract::NOperationResult::kDataError; } realOutStream.Release(); @@ -624,24 +620,14 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, const CLzxInfo &lzxInfo = section.Methods[0].LzxInfo; - if (!chmFolderOutStream) - { - chmFolderOutStream = new CChmFolderOutStream; - outStream = chmFolderOutStream; - } - chmFolderOutStream->Init(&m_Database, extractCallback, testMode); - if (!lzxDecoderSpec) - { - lzxDecoderSpec = new NCompress::NLzx::CDecoder; - lzxDecoder = lzxDecoderSpec; - } + lzxDecoder.Create_if_Empty(); UInt64 folderIndex = m_Database.GetFolder(index); const UInt64 compressedPos = m_Database.ContentOffset + section.Offset; - RINOK(lzxDecoderSpec->SetParams_and_Alloc(lzxInfo.GetNumDictBits())) + RINOK(lzxDecoder->Set_DictBits_and_Alloc(lzxInfo.GetNumDictBits())) const CItem *lastItem = &item; extractStatuses.Clear(); @@ -651,12 +637,12 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, { RINOK(extractCallback->SetCompleted(¤tTotalSize)) - UInt64 startPos = lzxInfo.GetFolderPos(folderIndex); + const UInt64 startPos = lzxInfo.GetFolderPos(folderIndex); UInt64 finishPos = lastItem->Offset + lastItem->Size; - UInt64 limitFolderIndex = lzxInfo.GetFolder(finishPos); + const UInt64 limitFolderIndex = lzxInfo.GetFolder(finishPos); lastFolderIndex = m_Database.GetLastFolder(index); - UInt64 folderSize = lzxInfo.GetFolderSize(); + const UInt64 folderSize = lzxInfo.GetFolderSize(); UInt64 unPackSize = folderSize; if (extractStatuses.IsEmpty()) @@ -697,9 +683,9 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, try { - UInt64 startBlock = lzxInfo.GetBlockIndexFromFolderIndex(folderIndex); + const UInt64 startBlock = lzxInfo.GetBlockIndexFromFolderIndex(folderIndex); const CResetTable &rt = lzxInfo.ResetTable; - UInt32 numBlocks = (UInt32)rt.GetNumBlocks(unPackSize); + const UInt32 numBlocks = (UInt32)rt.GetNumBlocks(unPackSize); for (UInt32 b = 0; b < numBlocks; b++) { @@ -708,33 +694,37 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, UInt64 bCur = startBlock + b; if (bCur >= rt.ResetOffsets.Size()) return E_FAIL; - UInt64 offset = rt.ResetOffsets[(unsigned)bCur]; + const UInt64 offset = rt.ResetOffsets[(unsigned)bCur]; UInt64 compressedSize; rt.GetCompressedSizeOfBlock(bCur, compressedSize); // chm writes full blocks. So we don't need to use reduced size for last block RINOK(InStream_SeekSet(m_Stream, compressedPos + offset)) - streamSpec->SetStream(m_Stream); - streamSpec->Init(compressedSize); + inStream->Init(compressedSize); - lzxDecoderSpec->SetKeepHistory(b > 0); + lzxDecoder->Set_KeepHistory(b > 0); - size_t compressedSizeT = (size_t)compressedSize; - if (compressedSizeT != compressedSize) - throw 2; - packBuf.AllocAtLeast(compressedSizeT); - - HRESULT res = ReadStream_FALSE(inStream, packBuf, compressedSizeT); - - if (res == S_OK) + HRESULT res = S_FALSE; + if (compressedSize <= (1u << 30)) { - lzxDecoderSpec->KeepHistoryForNext = true; - res = lzxDecoderSpec->Code(packBuf, compressedSizeT, kBlockSize); // rt.BlockSize; + const unsigned kAdditionalInputSize = 32; + const size_t compressedSizeT = (size_t)compressedSize; + const size_t allocSize = compressedSizeT + kAdditionalInputSize; + if (allocSize <= compressedSize) + throw 2; + packBuf.AllocAtLeast(allocSize); + res = ReadStream_FALSE(inStream, packBuf, compressedSizeT); if (res == S_OK) - res = WriteStream(chmFolderOutStream, - lzxDecoderSpec->GetUnpackData(), - lzxDecoderSpec->GetUnpackSize()); + { + memset(packBuf + compressedSizeT, 0xff, kAdditionalInputSize); + lzxDecoder->Set_KeepHistoryForNext(true); + res = lzxDecoder->Code_WithExceedReadWrite(packBuf, compressedSizeT, kBlockSize); // rt.BlockSize; + if (res == S_OK) + res = WriteStream(chmFolderOutStream, + lzxDecoder->GetUnpackData(), + lzxDecoder->GetUnpackSize()); + } } if (res != S_OK) diff --git a/CPP/7zip/Archive/Chm/ChmIn.cpp b/CPP/7zip/Archive/Chm/ChmIn.cpp index 28d512d..0866062 100644 --- a/CPP/7zip/Archive/Chm/ChmIn.cpp +++ b/CPP/7zip/Archive/Chm/ChmIn.cpp @@ -47,20 +47,15 @@ static bool inline AreGuidsEqual(const Byte *g1, const Byte *g2) return memcmp(g1, g2, 16) == 0; } -static char GetHex(unsigned v) +static void PrintByte(unsigned b, AString &s) { - return (char)((v < 10) ? ('0' + v) : ('A' + (v - 10))); -} - -static void PrintByte(Byte b, AString &s) -{ - s += GetHex(b >> 4); - s += GetHex(b & 0xF); + s += (char)GET_HEX_CHAR_UPPER(b >> 4); + s += (char)GET_HEX_CHAR_LOWER(b & 0xF); } AString CMethodInfo::GetGuidString() const { - char s[48]; + char s[16 * 2 + 8]; RawLeGuidToString_Braced(Guid, s); // MyStringUpper_Ascii(s); return (AString)s; @@ -93,12 +88,14 @@ AString CMethodInfo::GetName() const else { s = GetGuidString(); - if (ControlData.Size() > 0) + /* + if (ControlData.Size() > 0 && ControlData.Size() <= (1 << 6)) { - s += ':'; + s.Add_Colon(); for (size_t i = 0; i < ControlData.Size(); i++) PrintByte(ControlData[i], s); } + */ } } return s; @@ -177,7 +174,7 @@ UInt64 CInArchive::ReadEncInt() UInt64 val = 0; for (int i = 0; i < 9; i++) { - Byte b = ReadByte(); + const unsigned b = ReadByte(); val |= (b & 0x7F); if (b < 0x80) return val; @@ -206,7 +203,7 @@ void CInArchive::ReadUString(unsigned size, UString &s) s.Empty(); while (size-- != 0) { - wchar_t c = ReadUInt16(); + const wchar_t c = ReadUInt16(); if (c == 0) { Skip(2 * size); @@ -232,7 +229,7 @@ HRESULT CInArchive::ReadChunk(IInStream *inStream, UInt64 pos, UInt64 size) HRESULT CInArchive::ReadDirEntry(CDatabase &database) { CItem item; - UInt64 nameLen = ReadEncInt(); + const UInt64 nameLen = ReadEncInt(); if (nameLen == 0 || nameLen > (1 << 13)) return S_FALSE; ReadString((unsigned)nameLen, item.Name); @@ -333,7 +330,7 @@ HRESULT CInArchive::OpenChm(IInStream *inStream, CDatabase &database) // One quickref entry exists for every n entries in the file, where n // is calculated as 1 + (1 << quickref density). So for density = 2, n = 5. - UInt32 quickrefLength = ReadUInt32(); // Len of free space and/or quickref area at end of directory chunk + const UInt32 quickrefLength = ReadUInt32(); // Len of free space and/or quickref area at end of directory chunk if (quickrefLength > dirChunkSize || quickrefLength < 2) return S_FALSE; ReadUInt32(); // Always 0 @@ -345,8 +342,8 @@ HRESULT CInArchive::OpenChm(IInStream *inStream, CDatabase &database) for (;;) { - UInt64 offset = _inBuffer.GetProcessedSize() - chunkPos; - UInt32 offsetLimit = dirChunkSize - quickrefLength; + const UInt64 offset = _inBuffer.GetProcessedSize() - chunkPos; + const UInt32 offsetLimit = dirChunkSize - quickrefLength; if (offset > offsetLimit) return S_FALSE; if (offset == offsetLimit) @@ -357,7 +354,7 @@ HRESULT CInArchive::OpenChm(IInStream *inStream, CDatabase &database) Skip(quickrefLength - 2); - unsigned rrr = ReadUInt16(); + const unsigned rrr = ReadUInt16(); if (rrr != numItems) { // Lazarus 9-26-2 chm contains 0 here. @@ -538,34 +535,40 @@ HRESULT CInArchive::OpenHelp2(IInStream *inStream, CDatabase &database) unsigned numItems = 0; for (;;) { - UInt64 offset = _inBuffer.GetProcessedSize() - chunkPos; - UInt32 offsetLimit = dirChunkSize - quickrefLength; + const UInt64 offset = _inBuffer.GetProcessedSize() - chunkPos; + const UInt32 offsetLimit = dirChunkSize - quickrefLength; if (offset > offsetLimit) return S_FALSE; if (offset == offsetLimit) break; if (database.NewFormat) { - UInt16 nameLen = ReadUInt16(); + const unsigned nameLen = ReadUInt16(); if (nameLen == 0) return S_FALSE; UString name; - ReadUString((unsigned)nameLen, name); + ReadUString(nameLen, name); AString s; ConvertUnicodeToUTF8(name, s); - Byte b = ReadByte(); - s.Add_Space(); - PrintByte(b, s); + { + const unsigned b = ReadByte(); + s.Add_Space(); + PrintByte(b, s); + } s.Add_Space(); UInt64 len = ReadEncInt(); // then number of items ? // then length ? // then some data (binary encoding?) - while (len-- != 0) + if (len > 1u << 29) // what limit here we need? + return S_FALSE; + if (len) + do { - b = ReadByte(); + const unsigned b = ReadByte(); PrintByte(b, s); } + while (--len); database.NewFormatString += s; database.NewFormatString += "\r\n"; } @@ -613,7 +616,7 @@ static AString GetSectionPrefix(const AString &name) { AString s (kStorage); s += name; - s += '/'; + s.Add_Slash(); return s; } @@ -712,7 +715,7 @@ HRESULT CInArchive::OpenHighLevel(IInStream *inStream, CFilesDatabase &database) for (unsigned i = 0; i < numSections; i++) { CSectionInfo section; - UInt16 nameLen = ReadUInt16(); + const unsigned nameLen = ReadUInt16(); UString name; ReadUString(nameLen, name); if (ReadUInt16() != 0) @@ -799,14 +802,19 @@ HRESULT CInArchive::OpenHighLevel(IInStream *inStream, CFilesDatabase &database) li.CacheSize = ReadUInt32(); numDWORDS -= 5; - while (numDWORDS-- != 0) + if (numDWORDS) + do ReadUInt32(); + while (--numDWORDS); } else { - UInt32 numBytes = numDWORDS * 4; - method.ControlData.Alloc(numBytes); - ReadBytes(method.ControlData, numBytes); + if (numDWORDS > 1u << 27) + return S_FALSE; + const size_t numBytes = (size_t)numDWORDS * 4; + // method.ControlData.Alloc(numBytes); + // ReadBytes(method.ControlData, numBytes); + Skip(numBytes); } } } @@ -841,10 +849,10 @@ HRESULT CInArchive::OpenHighLevel(IInStream *inStream, CFilesDatabase &database) } else { - UInt32 ver = ReadUInt32(); // 2 unknown (possibly a version number) + const UInt32 ver = ReadUInt32(); // 2 unknown (possibly a version number) if (ver != 2 && ver != 3) return S_FALSE; - UInt32 numEntries = ReadUInt32(); + const UInt32 numEntries = ReadUInt32(); const unsigned kEntrySize = 8; if (ReadUInt32() != kEntrySize) return S_FALSE; diff --git a/CPP/7zip/Archive/Chm/ChmIn.h b/CPP/7zip/Archive/Chm/ChmIn.h index c01ef4d..faf4edf 100644 --- a/CPP/7zip/Archive/Chm/ChmIn.h +++ b/CPP/7zip/Archive/Chm/ChmIn.h @@ -45,9 +45,9 @@ struct CItem struct CDatabase { + CObjectVector Items; UInt64 StartPosition; UInt64 ContentOffset; - CObjectVector Items; AString NewFormatString; bool Help2Format; bool NewFormat; @@ -137,14 +137,14 @@ struct CLzxInfo return 0; } - UInt64 GetFolderSize() const { return kBlockSize << ResetIntervalBits; } + UInt64 GetFolderSize() const { return (UInt64)kBlockSize << ResetIntervalBits; } UInt64 GetFolder(UInt64 offset) const { return offset / GetFolderSize(); } UInt64 GetFolderPos(UInt64 folderIndex) const { return folderIndex * GetFolderSize(); } UInt64 GetBlockIndexFromFolderIndex(UInt64 folderIndex) const { return folderIndex << ResetIntervalBits; } bool GetOffsetOfFolder(UInt64 folderIndex, UInt64 &offset) const { - UInt64 blockIndex = GetBlockIndexFromFolderIndex(folderIndex); + const UInt64 blockIndex = GetBlockIndexFromFolderIndex(folderIndex); if (blockIndex >= ResetTable.ResetOffsets.Size()) return false; offset = ResetTable.ResetOffsets[(unsigned)blockIndex]; @@ -162,7 +162,7 @@ struct CLzxInfo struct CMethodInfo { Byte Guid[16]; - CByteBuffer ControlData; + // CByteBuffer ControlData; CLzxInfo LzxInfo; bool IsLzx() const; @@ -188,9 +188,9 @@ struct CSectionInfo class CFilesDatabase: public CDatabase { public: - bool LowLevel; CUIntVector Indices; CObjectVector Sections; + bool LowLevel; UInt64 GetFileSize(unsigned fileIndex) const { return Items[Indices[fileIndex]].Size; } UInt64 GetFileOffset(unsigned fileIndex) const { return Items[Indices[fileIndex]].Offset; } diff --git a/CPP/7zip/Archive/ComHandler.cpp b/CPP/7zip/Archive/ComHandler.cpp index 7aabd65..82d939d 100644 --- a/CPP/7zip/Archive/ComHandler.cpp +++ b/CPP/7zip/Archive/ComHandler.cpp @@ -252,9 +252,9 @@ static UString CompoundNameToFileName(const UString &s) const wchar_t c = s[i]; if ((unsigned)(int)c < 0x20) { - res += '['; + res.Add_Char('['); res.Add_UInt32((UInt32)(unsigned)(int)c); - res += ']'; + res.Add_Char(']'); } else res += c; @@ -309,20 +309,20 @@ static bool CompoundMsiNameToFileName(const UString &name, UString &res) if (i == 0) res += k_Msi_ID; */ - c -= k_Msi_StartUnicodeChar; + c -= (wchar_t)k_Msi_StartUnicodeChar; - unsigned c0 = (unsigned)c & k_Msi_CharMask; - unsigned c1 = (unsigned)c >> k_Msi_NumBits; + const unsigned c0 = (unsigned)c & k_Msi_CharMask; + const unsigned c1 = (unsigned)c >> k_Msi_NumBits; if (c1 <= k_Msi_NumChars) { - res += k_Msi_Chars[c0]; + res.Add_Char(k_Msi_Chars[c0]); if (c1 == k_Msi_NumChars) break; - res += k_Msi_Chars[c1]; + res.Add_Char(k_Msi_Chars[c1]); } else - res += k_Msi_SpecChar; + res.Add_Char(k_Msi_SpecChar); } return true; } diff --git a/CPP/7zip/Archive/Common/CoderMixer2.cpp b/CPP/7zip/Archive/Common/CoderMixer2.cpp index b6ddeb8..b1c316d 100644 --- a/CPP/7zip/Archive/Common/CoderMixer2.cpp +++ b/CPP/7zip/Archive/Common/CoderMixer2.cpp @@ -769,8 +769,8 @@ HRESULT CMixerST::Code( } } - const UInt64 * const *isSizes2 = EncodeMode ? &mainCoder.UnpackSizePointer : &mainCoder.PackSizePointers.Front(); - const UInt64 * const *outSizes2 = EncodeMode ? &mainCoder.PackSizePointers.Front() : &mainCoder.UnpackSizePointer; + const UInt64 * const *isSizes2 = EncodeMode ? &mainCoder.UnpackSizePointer : mainCoder.PackSizePointers.ConstData(); + const UInt64 * const *outSizes2 = EncodeMode ? mainCoder.PackSizePointers.ConstData() : &mainCoder.UnpackSizePointer; HRESULT res; if (mainCoder.Coder) @@ -783,8 +783,8 @@ HRESULT CMixerST::Code( else { res = mainCoder.Coder2->Code( - &seqInStreamsSpec.Front(), isSizes2, numInStreams, - &seqOutStreamsSpec.Front(), outSizes2, numOutStreams, + seqInStreamsSpec.ConstData(), isSizes2, numInStreams, + seqOutStreamsSpec.ConstData(), outSizes2, numOutStreams, progress); } @@ -909,8 +909,8 @@ void CCoderMT::Code(ICompressProgressInfo *progress) progress); else Result = Coder2->Code( - &InStreamPointers.Front(), EncodeMode ? &UnpackSizePointer : &PackSizePointers.Front(), numInStreams, - &OutStreamPointers.Front(), EncodeMode ? &PackSizePointers.Front(): &UnpackSizePointer, numOutStreams, + InStreamPointers.ConstData(), EncodeMode ? &UnpackSizePointer : PackSizePointers.ConstData(), numInStreams, + OutStreamPointers.ConstData(), EncodeMode ? PackSizePointers.ConstData(): &UnpackSizePointer, numOutStreams, progress); } diff --git a/CPP/7zip/Archive/Common/FindSignature.cpp b/CPP/7zip/Archive/Common/FindSignature.cpp index 6c6740f..4f46ee7 100644 --- a/CPP/7zip/Archive/Common/FindSignature.cpp +++ b/CPP/7zip/Archive/Common/FindSignature.cpp @@ -21,10 +21,10 @@ HRESULT FindSignatureInStream(ISequentialInStream *stream, if (memcmp(byteBuffer2, signature, signatureSize) == 0) return S_OK; - const UInt32 kBufferSize = (1 << 16); + const size_t kBufferSize = 1 << 16; CByteBuffer byteBuffer(kBufferSize); Byte *buffer = byteBuffer; - UInt32 numPrevBytes = signatureSize - 1; + size_t numPrevBytes = signatureSize - 1; memcpy(buffer, (const Byte *)byteBuffer2 + 1, numPrevBytes); resPos = 1; for (;;) @@ -34,16 +34,16 @@ HRESULT FindSignatureInStream(ISequentialInStream *stream, return S_FALSE; do { - const UInt32 numReadBytes = kBufferSize - numPrevBytes; + const size_t numReadBytes = kBufferSize - numPrevBytes; UInt32 processedSize; - RINOK(stream->Read(buffer + numPrevBytes, numReadBytes, &processedSize)) - numPrevBytes += processedSize; + RINOK(stream->Read(buffer + numPrevBytes, (UInt32)numReadBytes, &processedSize)) + numPrevBytes += (size_t)processedSize; if (processedSize == 0) return S_FALSE; } while (numPrevBytes < signatureSize); - const UInt32 numTests = numPrevBytes - signatureSize + 1; - for (UInt32 pos = 0; pos < numTests; pos++) + const size_t numTests = numPrevBytes - signatureSize + 1; + for (size_t pos = 0; pos < numTests; pos++) { const Byte b = signature[0]; for (; buffer[pos] != b && pos < numTests; pos++); @@ -60,3 +60,31 @@ HRESULT FindSignatureInStream(ISequentialInStream *stream, memmove(buffer, buffer + numTests, numPrevBytes); } } + +namespace NArchive { +HRESULT ReadZeroTail(ISequentialInStream *stream, bool &areThereNonZeros, UInt64 &numZeros, UInt64 maxSize); +HRESULT ReadZeroTail(ISequentialInStream *stream, bool &areThereNonZeros, UInt64 &numZeros, UInt64 maxSize) +{ + areThereNonZeros = false; + numZeros = 0; + const size_t kBufSize = 1 << 11; + Byte buf[kBufSize]; + for (;;) + { + UInt32 size = 0; + RINOK(stream->Read(buf, kBufSize, &size)) + if (size == 0) + return S_OK; + for (UInt32 i = 0; i < size; i++) + if (buf[i] != 0) + { + areThereNonZeros = true; + numZeros += i; + return S_OK; + } + numZeros += size; + if (numZeros > maxSize) + return S_OK; + } +} +} diff --git a/CPP/7zip/Archive/Common/InStreamWithCRC.h b/CPP/7zip/Archive/Common/InStreamWithCRC.h index 2a91d76..4873689 100644 --- a/CPP/7zip/Archive/Common/InStreamWithCRC.h +++ b/CPP/7zip/Archive/Common/InStreamWithCRC.h @@ -40,12 +40,9 @@ Z7_CLASS_IMP_NOQIB_2( }; -Z7_CLASS_IMP_COM_1( - CInStreamWithCRC, - IInStream +Z7_CLASS_IMP_IInStream( + CInStreamWithCRC ) - Z7_IFACE_COM7_IMP(ISequentialInStream) - CMyComPtr _stream; UInt64 _size; UInt32 _crc; diff --git a/CPP/7zip/Archive/Common/MultiStream.h b/CPP/7zip/Archive/Common/MultiStream.h index e3096f5..901b6ea 100644 --- a/CPP/7zip/Archive/Common/MultiStream.h +++ b/CPP/7zip/Archive/Common/MultiStream.h @@ -9,11 +9,9 @@ #include "../../IStream.h" #include "../../Archive/IArchive.h" -Z7_CLASS_IMP_COM_1( +Z7_CLASS_IMP_IInStream( CMultiStream - , IInStream ) - Z7_IFACE_COM7_IMP(ISequentialInStream) unsigned _streamIndex; UInt64 _pos; diff --git a/CPP/7zip/Archive/Common/OutStreamWithSha1.cpp b/CPP/7zip/Archive/Common/OutStreamWithSha1.cpp index 57c18ec..2916c30 100644 --- a/CPP/7zip/Archive/Common/OutStreamWithSha1.cpp +++ b/CPP/7zip/Archive/Common/OutStreamWithSha1.cpp @@ -16,3 +16,14 @@ Z7_COM7F_IMF(COutStreamWithSha1::Write(const void *data, UInt32 size, UInt32 *pr *processedSize = size; return result; } + +Z7_COM7F_IMF(CInStreamWithSha1::Read(void *data, UInt32 size, UInt32 *processedSize)) +{ + UInt32 realProcessedSize; + const HRESULT result = _stream->Read(data, size, &realProcessedSize); + _size += realProcessedSize; + Sha1_Update(Sha(), (const Byte *)data, realProcessedSize); + if (processedSize) + *processedSize = realProcessedSize; + return result; +} diff --git a/CPP/7zip/Archive/Common/OutStreamWithSha1.h b/CPP/7zip/Archive/Common/OutStreamWithSha1.h index 74bc53c..28b2587 100644 --- a/CPP/7zip/Archive/Common/OutStreamWithSha1.h +++ b/CPP/7zip/Archive/Common/OutStreamWithSha1.h @@ -14,11 +14,10 @@ Z7_CLASS_IMP_NOQIB_1( COutStreamWithSha1 , ISequentialOutStream ) - CMyComPtr _stream; - UInt64 _size; - // CSha1 _sha; bool _calculate; + CMyComPtr _stream; CAlignedBuffer1 _sha; + UInt64 _size; CSha1 *Sha() { return (CSha1 *)(void *)(Byte *)_sha; } public: @@ -27,8 +26,8 @@ Z7_CLASS_IMP_NOQIB_1( void ReleaseStream() { _stream.Release(); } void Init(bool calculate = true) { - _size = 0; _calculate = calculate; + _size = 0; Sha1_Init(Sha()); } void InitSha1() { Sha1_Init(Sha()); } @@ -36,4 +35,27 @@ Z7_CLASS_IMP_NOQIB_1( void Final(Byte *digest) { Sha1_Final(Sha(), digest); } }; + +Z7_CLASS_IMP_NOQIB_1( + CInStreamWithSha1 + , ISequentialInStream +) + CMyComPtr _stream; + CAlignedBuffer1 _sha; + UInt64 _size; + + CSha1 *Sha() { return (CSha1 *)(void *)(Byte *)_sha; } +public: + CInStreamWithSha1(): _sha(sizeof(CSha1)) {} + void SetStream(ISequentialInStream *stream) { _stream = stream; } + void Init() + { + _size = 0; + Sha1_Init(Sha()); + } + void ReleaseStream() { _stream.Release(); } + UInt64 GetSize() const { return _size; } + void Final(Byte *digest) { Sha1_Final(Sha(), digest); } +}; + #endif diff --git a/CPP/7zip/Archive/CpioHandler.cpp b/CPP/7zip/Archive/CpioHandler.cpp index 872cea7..2228040 100644 --- a/CPP/7zip/Archive/CpioHandler.cpp +++ b/CPP/7zip/Archive/CpioHandler.cpp @@ -30,8 +30,6 @@ namespace NCpio { static const Byte kMagicBin0 = 0xC7; static const Byte kMagicBin1 = 0x71; -// #define MAGIC_ASCII { '0', '7', '0', '7', '0' } - static const Byte kMagicHex = '1'; // New ASCII Format static const Byte kMagicHexCrc = '2'; // New CRC Format static const Byte kMagicOct = '7'; // Portable ASCII Format @@ -147,8 +145,8 @@ static bool CheckOctRecord(const Byte *p) { for (unsigned i = 6; i < k_OctRecord_Size; i++) { - const Byte c = p[i]; - if (c < '0' || c > '7') + const unsigned c = (unsigned)p[i] - '0'; + if (c > 7) return false; } return true; @@ -158,11 +156,15 @@ static bool CheckHexRecord(const Byte *p) { for (unsigned i = 6; i < k_HexRecord_Size; i++) { - const Byte c = p[i]; - if ((c < '0' || c > '9') && - (c < 'A' || c > 'F') && - (c < 'a' || c > 'f')) - return false; + unsigned c = p[i]; + c -= '0'; + if (c > 9) + { + c -= 'A' - '0'; + c &= ~0x20u; + if (c > 5) + return false; + } } return true; } @@ -214,7 +216,7 @@ API_FUNC_static_IsArc IsArc_Cpio(const Byte *p, size_t size) UInt32 namePos; UInt32 nameSize; UInt32 mode; - UInt32 rDevMinor; + // UInt32 rDevMinor; UInt32 rDevMajor = 0; if (p[0] == '0') @@ -231,7 +233,7 @@ API_FUNC_static_IsArc IsArc_Cpio(const Byte *p, size_t size) if (!CheckOctRecord(p)) return k_IsArc_Res_NO; READ_OCT_6 (2 * 6, mode) - READ_OCT_6 (6 * 6, rDevMinor) + // READ_OCT_6 (6 * 6, rDevMinor) READ_OCT_6 (7 * 6 + 11, nameSize) namePos = k_OctRecord_Size; } @@ -243,7 +245,7 @@ API_FUNC_static_IsArc IsArc_Cpio(const Byte *p, size_t size) return k_IsArc_Res_NO; READ_HEX (1, mode) READ_HEX (9, rDevMajor) - READ_HEX (10, rDevMinor) + // READ_HEX (10, rDevMinor) READ_HEX (11, nameSize) namePos = k_HexRecord_Size; } @@ -255,13 +257,13 @@ API_FUNC_static_IsArc IsArc_Cpio(const Byte *p, size_t size) if (p[0] == kMagicBin0 && p[1] == kMagicBin1) { mode = GetUi16(p + 6); - rDevMinor = GetUi16(p + 14); + // rDevMinor = GetUi16(p + 14); nameSize = GetUi16(p + 20); } else if (p[0] == kMagicBin1 && p[1] == kMagicBin0) { mode = GetBe16(p + 6); - rDevMinor = GetBe16(p + 14); + // rDevMinor = GetBe16(p + 14); nameSize = GetBe16(p + 20); } else @@ -272,8 +274,11 @@ API_FUNC_static_IsArc IsArc_Cpio(const Byte *p, size_t size) if (mode >= (1 << 16)) return k_IsArc_Res_NO; - if (rDevMajor != 0 || - rDevMinor != 0) + /* v23.02: we have disabled rDevMinor check because real file + from Apple contains rDevMinor==255 by some unknown reason */ + if (rDevMajor != 0 + // || rDevMinor != 0 + ) { if (!MY_LIN_S_ISCHR(mode) && !MY_LIN_S_ISBLK(mode)) @@ -431,8 +436,11 @@ HRESULT CInArchive::GetNextItem() if (item.Mode >= (1 << 16)) return S_OK; - if (item.RDevMinor != 0 || - item.RDevMajor != 0) + /* v23.02: we have disabled rDevMinor check because real file + from Apple contains rDevMinor==255 by some unknown reason */ + if (item.RDevMajor != 0 + // || item.RDevMinor != 0 + ) { if (!MY_LIN_S_ISCHR(item.Mode) && !MY_LIN_S_ISBLK(item.Mode)) @@ -954,38 +962,48 @@ Z7_CLASS_IMP_NOQIB_1( , ISequentialOutStream ) CMyComPtr _stream; - UInt64 _size; - UInt32 _crc; + UInt32 _checksum; bool _calculate; public: void SetStream(ISequentialOutStream *stream) { _stream = stream; } void ReleaseStream() { _stream.Release(); } - void Init(bool calculate = true) + void Init(bool calculate) { - _size = 0; _calculate = calculate; - _crc = 0; + _checksum = 0; } - void EnableCalc(bool calculate) { _calculate = calculate; } - void InitCRC() { _crc = 0; } - UInt64 GetSize() const { return _size; } - UInt32 GetCRC() const { return _crc; } + UInt32 GetChecksum() const { return _checksum; } }; + Z7_COM7F_IMF(COutStreamWithSum::Write(const void *data, UInt32 size, UInt32 *processedSize)) { HRESULT result = S_OK; if (_stream) result = _stream->Write(data, size, &size); + if (processedSize) + *processedSize = size; if (_calculate) { - UInt32 crc = 0; - for (UInt32 i = 0; i < size; i++) - crc += (UInt32)(((const Byte *)data)[i]); - _crc += crc; + const Byte *p = (const Byte *)data; + const Byte *lim = p + size; + UInt32 sum = _checksum; + if (size >= 4) + { + lim -= 4 - 1; + do + { + sum += p[0] + p[1] + p[2] + p[3]; + p += 4; + } + while (p < lim); + lim += 4 - 1; + } + if (p != lim) { sum += *p++; + if (p != lim) { sum += *p++; + if (p != lim) { sum += *p++; }}} + _checksum = sum; } - if (processedSize) - *processedSize = size; return result; } @@ -1009,19 +1027,12 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, } RINOK(extractCallback->SetTotal(totalSize)) - NCompress::CCopyCoder *copyCoderSpec = new NCompress::CCopyCoder(); - CMyComPtr copyCoder = copyCoderSpec; - - CLocalProgress *lps = new CLocalProgress; - CMyComPtr progress = lps; + CMyComPtr2_Create copyCoder; + CMyComPtr2_Create lps; lps->Init(extractCallback, false); - - CLimitedSequentialInStream *streamSpec = new CLimitedSequentialInStream; - CMyComPtr inStream(streamSpec); - streamSpec->SetStream(_stream); - - COutStreamWithSum *outStreamSumSpec = new COutStreamWithSum; - CMyComPtr outStreamSum(outStreamSumSpec); + CMyComPtr2_Create inStream; + inStream->SetStream(_stream); + CMyComPtr2_Create outStreamSum; UInt64 total_PackSize = 0; UInt64 total_UnpackSize = 0; @@ -1033,40 +1044,40 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, RINOK(lps->SetCur()) if (i >= numItems) break; - CMyComPtr outStream; const Int32 askMode = testMode ? NExtract::NAskMode::kTest : NExtract::NAskMode::kExtract; const UInt32 index = allFilesMode ? i : indices[i]; const CItem &item2 = _items[index]; const CItem &item = _items[item2.MainIndex_ForInode]; - RINOK(extractCallback->GetStream(index, &outStream, askMode)) - - total_PackSize += item2.GetPackSize(); - total_UnpackSize += item.Size; - - if (item2.IsDir()) { + CMyComPtr outStream; + RINOK(extractCallback->GetStream(index, &outStream, askMode)) + + total_PackSize += item2.GetPackSize(); + total_UnpackSize += item.Size; + + if (item2.IsDir()) + { + RINOK(extractCallback->PrepareOperation(askMode)) + RINOK(extractCallback->SetOperationResult(NExtract::NOperationResult::kOK)) + continue; + } + if (!testMode && !outStream) + continue; + outStreamSum->Init(item.IsCrcFormat()); + outStreamSum->SetStream(outStream); RINOK(extractCallback->PrepareOperation(askMode)) - RINOK(extractCallback->SetOperationResult(NExtract::NOperationResult::kOK)) - continue; } - if (!testMode && !outStream) - continue; - outStreamSumSpec->Init(item.IsCrcFormat()); - outStreamSumSpec->SetStream(outStream); - outStream.Release(); - - RINOK(extractCallback->PrepareOperation(askMode)) RINOK(InStream_SeekSet(_stream, item.GetDataPosition())) - streamSpec->Init(item.Size); - RINOK(copyCoder->Code(inStream, outStreamSum, NULL, NULL, progress)) - outStreamSumSpec->ReleaseStream(); + inStream->Init(item.Size); + RINOK(copyCoder.Interface()->Code(inStream, outStreamSum, NULL, NULL, lps)) + outStreamSum->ReleaseStream(); Int32 res = NExtract::NOperationResult::kDataError; - if (copyCoderSpec->TotalSize == item.Size) + if (copyCoder->TotalSize == item.Size) { res = NExtract::NOperationResult::kOK; - if (item.IsCrcFormat() && item.ChkSum != outStreamSumSpec->GetCRC()) + if (item.IsCrcFormat() && item.ChkSum != outStreamSum->GetChecksum()) res = NExtract::NOperationResult::kCRCError; } RINOK(extractCallback->SetOperationResult(res)) diff --git a/CPP/7zip/Archive/DmgHandler.cpp b/CPP/7zip/Archive/DmgHandler.cpp index 4bcb904..3901192 100644 --- a/CPP/7zip/Archive/DmgHandler.cpp +++ b/CPP/7zip/Archive/DmgHandler.cpp @@ -2,10 +2,13 @@ #include "StdAfx.h" +#include "../../../C/Alloc.h" #include "../../../C/CpuArch.h" +#include "../../Common/AutoPtr.h" #include "../../Common/ComTry.h" #include "../../Common/IntToString.h" +#include "../../Common/MyBuffer2.h" #include "../../Common/MyXml.h" #include "../../Common/UTFConvert.h" @@ -20,32 +23,67 @@ #include "../Compress/BZip2Decoder.h" #include "../Compress/CopyCoder.h" #include "../Compress/LzfseDecoder.h" +#include "../Compress/XzDecoder.h" #include "../Compress/ZlibDecoder.h" #include "Common/OutStreamWithCRC.h" // #define DMG_SHOW_RAW -// #include -#define PRF(x) // x +// #define SHOW_DEBUG_INFO + +/* for debug only: we can use block cache also for METHOD_COPY blocks. + It can reduce the number of Stream->Read() requests. + But it can increase memory usage. + Note: METHOD_COPY blocks are not fused usually. + But if METHOD_COPY blocks is fused in some dmg example, + block size can exceed k_Chunk_Size_MAX. + So we don't use cache for METHOD_COPY block, if (block_size > k_Chunk_Size_MAX) +*/ +// for debug only: +// #define Z7_DMG_USE_CACHE_FOR_COPY_BLOCKS + +#ifdef SHOW_DEBUG_INFO +#include +#define PRF(x) x +#else +#define PRF(x) +#endif + #define Get16(p) GetBe16(p) #define Get32(p) GetBe32(p) #define Get64(p) GetBe64(p) +#define Get32a(p) GetBe32a(p) +#define Get64a(p) GetBe64a(p) + Byte *Base64ToBin(Byte *dest, const char *src); namespace NArchive { namespace NDmg { +// allocation limits for compressed blocks for GetStream() interface: +static const unsigned k_NumChunks_MAX = 128; +static const size_t k_Chunk_Size_MAX = (size_t)1 << 28; +// 256 MB cache for 32-bit: +// 4 GB cache for 64-bit: +static const size_t k_Chunks_TotalSize_MAX = (size_t)1 << (sizeof(size_t) + 24); + +// 2 GB limit for 32-bit: +// 4 GB limit for 64-bit: +// that limit can be increased for 64-bit mode, if there are such dmg files +static const size_t k_XmlSize_MAX = + ((size_t)1 << (sizeof(size_t) / 4 + 30)) - 256; -static const UInt32 METHOD_ZERO_0 = 0; +static const UInt32 METHOD_ZERO_0 = 0; // sparse static const UInt32 METHOD_COPY = 1; -static const UInt32 METHOD_ZERO_2 = 2; // without file CRC calculation +static const UInt32 METHOD_ZERO_2 = 2; // sparse : without file CRC calculation static const UInt32 METHOD_ADC = 0x80000004; static const UInt32 METHOD_ZLIB = 0x80000005; static const UInt32 METHOD_BZIP2 = 0x80000006; static const UInt32 METHOD_LZFSE = 0x80000007; +static const UInt32 METHOD_XZ = 0x80000008; static const UInt32 METHOD_COMMENT = 0x7FFFFFFE; // is used to comment "+beg" and "+end" in extra field. static const UInt32 METHOD_END = 0xFFFFFFFF; @@ -54,20 +92,41 @@ struct CBlock { UInt32 Type; UInt64 UnpPos; - UInt64 UnpSize; + // UInt64 UnpSize; UInt64 PackPos; UInt64 PackSize; - UInt64 GetNextPackOffset() const { return PackPos + PackSize; } - UInt64 GetNextUnpPos() const { return UnpPos + UnpSize; } + bool NeedCrc() const { return Type != METHOD_ZERO_2; } + bool IsZeroMethod() const + { + return (Type & ~(UInt32)METHOD_ZERO_2) == 0; + // return Type == METHOD_ZERO_0 || Type == METHOD_ZERO_2; + } + + bool IsClusteredMethod() const + { + // most of dmg files have non-fused COPY_METHOD blocks. + // so we don't exclude COPY_METHOD blocks when we try to detect size of cluster. + return !IsZeroMethod(); // include COPY_METHOD blocks. + // Type > METHOD_ZERO_2; // for debug: exclude COPY_METHOD blocks. + } - bool IsZeroMethod() const { return Type == METHOD_ZERO_0 || Type == METHOD_ZERO_2; } - bool ThereAreDataInBlock() const { return Type != METHOD_COMMENT && Type != METHOD_END; } + bool NeedAllocateBuffer() const + { + return +#ifdef Z7_DMG_USE_CACHE_FOR_COPY_BLOCKS + !IsZeroMethod(); +#else + Type > METHOD_ZERO_2; + // !IsZeroMethod() && Type != METHOD_COPY; +#endif + } + // we don't store non-data blocks now + // bool ThereAreDataInBlock() const { return Type != METHOD_COMMENT && Type != METHOD_END; } }; static const UInt32 kCheckSumType_CRC = 2; - -static const size_t kChecksumSize_Max = 0x80; +static const unsigned kChecksumSize_Max = 0x80; struct CChecksum { @@ -78,6 +137,11 @@ struct CChecksum bool IsCrc32() const { return Type == kCheckSumType_CRC && NumBits == 32; } UInt32 GetCrc32() const { return Get32(Data); } void Parse(const Byte *p); + + void PrintType(AString &s) const; + void Print(AString &s) const; + void Print_with_Name(AString &s) const; + void AddToComment(AString &s, const char *name) const; }; void CChecksum::Parse(const Byte *p) @@ -87,16 +151,107 @@ void CChecksum::Parse(const Byte *p) memcpy(Data, p + 8, kChecksumSize_Max); } + +void CChecksum::PrintType(AString &s) const +{ + if (NumBits == 0) + return; + if (IsCrc32()) + s += "CRC"; + else + { + s += "Checksum"; + s.Add_UInt32(Type); + s.Add_Minus(); + s.Add_UInt32(NumBits); + } +} + +void CChecksum::Print(AString &s) const +{ + if (NumBits == 0) + return; + char temp[kChecksumSize_Max * 2 + 2]; + /* + if (IsCrc32()) + ConvertUInt32ToHex8Digits(GetCrc32(), temp); + else + */ + { + UInt32 numBits = kChecksumSize_Max * 8; + if (numBits > NumBits) + numBits = NumBits; + const unsigned numBytes = (numBits + 7) >> 3; + if (numBytes <= 8) + ConvertDataToHex_Upper(temp, Data, numBytes); + else + ConvertDataToHex_Lower(temp, Data, numBytes); + } + s += temp; +} + +void CChecksum::Print_with_Name(AString &s) const +{ + if (NumBits == 0) + return; + PrintType(s); + s += ": "; + Print(s); +} + +static void AddToComment_Prop(AString &s, const char *name, const char *val) +{ + s += name; + s += ": "; + s += val; + s.Add_LF(); +} + +void CChecksum::AddToComment(AString &s, const char *name) const +{ + AString s2; + Print_with_Name(s2); + if (!s2.IsEmpty()) + AddToComment_Prop(s, name, s2); +} + + struct CFile { UInt64 Size; + CRecordVector Blocks; UInt64 PackSize; - UInt64 StartPos; + UInt64 StartPackPos; + UInt64 BlockSize_MAX; + UInt64 StartUnpackSector; // unpack sector position of this file from all files + UInt64 NumUnpackSectors; + Int32 Descriptor; + bool IsCorrect; + bool FullFileChecksum; AString Name; - CRecordVector Blocks; + // AString Id; CChecksum Checksum; - bool FullFileChecksum; + UInt64 GetUnpackSize_of_Block(unsigned blockIndex) const + { + return (blockIndex == Blocks.Size() - 1 ? + Size : Blocks[blockIndex + 1].UnpPos) - Blocks[blockIndex].UnpPos; + } + + CFile(): + Size(0), + PackSize(0), + StartPackPos(0), + BlockSize_MAX(0), + StartUnpackSector(0), + NumUnpackSectors(0), + Descriptor(0), + IsCorrect(false), + FullFileChecksum(false) + { + Checksum.Type = 0; + Checksum.NumBits = 0; + } HRESULT Parse(const Byte *p, UInt32 size); }; @@ -109,87 +264,127 @@ struct CExtraFile #endif +static void AddToComment_UInt64(AString &s, UInt64 v, const char *name) +{ + s += name; + s += ": "; + s.Add_UInt64(v); + s.Add_LF(); +} + struct CForkPair { UInt64 Offset; UInt64 Len; + // (p) is aligned for 8-bytes void Parse(const Byte *p) { - Offset = Get64(p); - Len = Get64(p + 8); + Offset = Get64a(p); + Len = Get64a(p + 8); + } + + bool GetEndPos(UInt64 &endPos) + { + endPos = Offset + Len; + return endPos >= Offset; } bool UpdateTop(UInt64 limit, UInt64 &top) { if (Offset > limit || Len > limit - Offset) return false; - UInt64 top2 = Offset + Len; + const UInt64 top2 = Offset + Len; if (top <= top2) top = top2; return true; } + + void Print(AString &s, const char *name) const; }; +void CForkPair::Print(AString &s, const char *name) const +{ + if (Offset != 0 || Len != 0) + { + s += name; s.Add_Minus(); AddToComment_UInt64(s, Offset, "offset"); + s += name; s.Add_Minus(); AddToComment_UInt64(s, Len, "length"); + } +} + Z7_CLASS_IMP_CHandler_IInArchive_1( IInArchiveGetStream ) - CMyComPtr _inStream; - CObjectVector _files; bool _masterCrcError; bool _headersError; + bool _dataForkError; + bool _rsrcMode_wasUsed; + + CMyComPtr _inStream; + CObjectVector _files; - UInt32 _dataStartOffset; + // UInt32 _dataStartOffset; UInt64 _startPos; UInt64 _phySize; AString _name; - - #ifdef DMG_SHOW_RAW + + CForkPair _dataForkPair; + CForkPair rsrcPair, xmlPair, blobPair; + + // UInt64 _runningDataForkOffset; + // UInt32 _segmentNumber; + // UInt32 _segmentCount; + UInt64 _numSectors; // total unpacked number of sectors + Byte _segmentGUID[16]; + + CChecksum _dataForkChecksum; + CChecksum _masterChecksum; + +#ifdef DMG_SHOW_RAW CObjectVector _extras; - #endif +#endif HRESULT ReadData(IInStream *stream, const CForkPair &pair, CByteBuffer &buf); bool ParseBlob(const CByteBuffer &data); - HRESULT Open2(IInStream *stream); + HRESULT Open2(IInStream *stream, IArchiveOpenCallback *openArchiveCallback); HRESULT Extract(IInStream *stream); }; -// that limit can be increased, if there are such dmg files -static const size_t kXmlSizeMax = 0xFFFF0000; // 4 GB - 64 KB; struct CMethods { CRecordVector Types; - CRecordVector ChecksumTypes; void Update(const CFile &file); - void GetString(AString &s) const; + void AddToString(AString &s) const; }; void CMethods::Update(const CFile &file) { - ChecksumTypes.AddToUniqueSorted(file.Checksum.Type); FOR_VECTOR (i, file.Blocks) + { + if (Types.Size() >= (1 << 8)) + break; Types.AddToUniqueSorted(file.Blocks[i].Type); + } } -void CMethods::GetString(AString &res) const +void CMethods::AddToString(AString &res) const { - res.Empty(); - - unsigned i; - - for (i = 0; i < Types.Size(); i++) + FOR_VECTOR (i, Types) { const UInt32 type = Types[i]; + /* if (type == METHOD_COMMENT || type == METHOD_END) continue; + */ char buf[16]; const char *s; switch (type) { + // case METHOD_COMMENT: s = "Comment"; break; case METHOD_ZERO_0: s = "Zero0"; break; case METHOD_ZERO_2: s = "Zero2"; break; case METHOD_COPY: s = "Copy"; break; @@ -197,25 +392,15 @@ void CMethods::GetString(AString &res) const case METHOD_ZLIB: s = "ZLIB"; break; case METHOD_BZIP2: s = "BZip2"; break; case METHOD_LZFSE: s = "LZFSE"; break; - default: ConvertUInt32ToString(type, buf); s = buf; + case METHOD_XZ: s = "XZ"; break; + default: ConvertUInt32ToHex(type, buf); s = buf; } res.Add_OptSpaced(s); } - - for (i = 0; i < ChecksumTypes.Size(); i++) - { - res.Add_Space_if_NotEmpty(); - UInt32 type = ChecksumTypes[i]; - switch (type) - { - case kCheckSumType_CRC: res += "CRC"; break; - default: - res += "Check"; - res.Add_UInt32(type); - } - } } + + struct CAppleName { bool IsFs; @@ -229,6 +414,7 @@ static const CAppleName k_Names[] = { true, "hfsx", "Apple_HFSX" }, { true, "ufs", "Apple_UFS" }, { true, "apfs", "Apple_APFS" }, + { true, "iso", "Apple_ISO" }, // efi_sys partition is FAT32, but it's not main file. So we use (IsFs = false) { false, "efi_sys", "C12A7328-F81F-11D2-BA4B-00A0C93EC93B" }, @@ -237,6 +423,13 @@ static const CAppleName k_Names[] = { false, "ddm", "DDM" }, { false, NULL, "Apple_partition_map" }, { false, NULL, " GPT " }, + /* " GPT " is substring of full name entry in dmg that contains + some small metadata GPT entry. It's not real FS entry: + "Primary GPT Header", + "Primary GPT Table" + "Backup GPT Header", + "Backup GPT Table", + */ { false, NULL, "MBR" }, { false, NULL, "Driver" }, { false, NULL, "Patches" } @@ -244,15 +437,54 @@ static const CAppleName k_Names[] = static const unsigned kNumAppleNames = Z7_ARRAY_SIZE(k_Names); +const char *Find_Apple_FS_Ext(const AString &name); +const char *Find_Apple_FS_Ext(const AString &name) +{ + for (unsigned i = 0; i < kNumAppleNames; i++) + { + const CAppleName &a = k_Names[i]; + if (a.Ext) + if (name == a.AppleName) + return a.Ext; + } + return NULL; +} + + +bool Is_Apple_FS_Or_Unknown(const AString &name); +bool Is_Apple_FS_Or_Unknown(const AString &name) +{ + for (unsigned i = 0; i < kNumAppleNames; i++) + { + const CAppleName &a = k_Names[i]; + // if (name.Find(a.AppleName) >= 0) + if (strstr(name, a.AppleName)) + return a.IsFs; + } + return true; +} + + + +// define it for debug only: +// #define Z7_DMG_SINGLE_FILE_MODE + static const Byte kProps[] = { kpidPath, kpidSize, kpidPackSize, - kpidCRC, kpidComment, - kpidMethod + kpidMethod, + kpidNumBlocks, + kpidClusterSize, + kpidChecksum, + kpidCRC, + kpidId // kpidOffset +#ifdef Z7_DMG_SINGLE_FILE_MODE + , kpidPosition +#endif }; IMP_IInArchive_Props @@ -261,9 +493,11 @@ static const Byte kArcProps[] = { kpidMethod, kpidNumBlocks, + kpidClusterSize, kpidComment }; + Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) { COM_TRY_BEGIN @@ -273,10 +507,33 @@ Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) case kpidMethod: { CMethods m; - FOR_VECTOR (i, _files) - m.Update(_files[i]); + CRecordVector ChecksumTypes; + { + FOR_VECTOR (i, _files) + { + const CFile &file = _files[i]; + m.Update(file); + if (ChecksumTypes.Size() < (1 << 8)) + ChecksumTypes.AddToUniqueSorted(file.Checksum.Type); + } + } AString s; - m.GetString(s); + m.AddToString(s); + + FOR_VECTOR (i, ChecksumTypes) + { + const UInt32 type = ChecksumTypes[i]; + switch (type) + { + case kCheckSumType_CRC: + s.Add_OptSpaced("CRC"); + break; + default: + s.Add_OptSpaced("Checksum"); + s.Add_UInt32(type); + } + } + if (!s.IsEmpty()) prop = s; break; @@ -289,36 +546,34 @@ Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) prop = numBlocks; break; } + case kpidClusterSize: + { + UInt64 blockSize_MAX = 0; + FOR_VECTOR (i, _files) + { + const UInt64 a = _files[i].BlockSize_MAX; + if (blockSize_MAX < a) + blockSize_MAX = a; + } + prop = blockSize_MAX; + break; + } case kpidMainSubfile: { int mainIndex = -1; - unsigned numFS = 0; - unsigned numUnknown = 0; FOR_VECTOR (i, _files) { - const AString &name = _files[i].Name; - unsigned n; - for (n = 0; n < kNumAppleNames; n++) + if (Is_Apple_FS_Or_Unknown(_files[i].Name)) { - const CAppleName &appleName = k_Names[n]; - // if (name.Find(appleName.AppleName) >= 0) - if (strstr(name, appleName.AppleName)) + if (mainIndex != -1) { - if (appleName.IsFs) - { - numFS++; - mainIndex = (int)i; - } + mainIndex = -1; break; } - } - if (n == kNumAppleNames) - { mainIndex = (int)i; - numUnknown++; } } - if (numFS + numUnknown == 1) + if (mainIndex != -1) prop = (UInt32)(Int32)mainIndex; break; } @@ -330,7 +585,8 @@ Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) case kpidWarningFlags: { UInt32 v = 0; - if (_headersError) v |= kpv_ErrorFlags_HeadersError; + if (_headersError) v |= kpv_ErrorFlags_HeadersError; + if (_dataForkError) v |= kpv_ErrorFlags_CrcError; if (v != 0) prop = v; break; @@ -340,15 +596,39 @@ Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) case kpidPhySize: prop = _phySize; break; case kpidComment: - if (!_name.IsEmpty() && _name.Len() < 256) - prop = _name; - break; + { + AString s; + if (!_name.IsEmpty()) + AddToComment_Prop(s, "Name", _name); + AddToComment_UInt64(s, _numSectors << 9, "unpack-size"); + { + char temp[sizeof(_segmentGUID) * 2 + 2]; + ConvertDataToHex_Lower(temp, _segmentGUID, sizeof(_segmentGUID)); + AddToComment_Prop(s, "ID", temp); + } + _masterChecksum.AddToComment(s, "master-checksum"); + _dataForkChecksum.AddToComment(s, "pack-checksum"); + { + /* + if (_dataStartOffset != 0) + AddToComment_UInt64(s, _dataStartOffset, "payload-start-offset"); + */ + // if (_dataForkPair.Offset != 0) + _dataForkPair.Print(s, "pack"); + rsrcPair.Print(s, "rsrc"); + xmlPair.Print(s, "xml"); + blobPair.Print(s, "blob"); + } + if (_rsrcMode_wasUsed) + s += "RSRC_MODE\n"; + if (!s.IsEmpty()) + prop = s; + } + break; case kpidName: - if (!_name.IsEmpty() && _name.Len() < 256) - { + if (!_name.IsEmpty()) prop = _name + ".dmg"; - } break; } prop.Detach(value); @@ -358,39 +638,64 @@ Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) IMP_IInArchive_ArcProps + + +static const UInt64 kSectorNumber_LIMIT = (UInt64)1 << (63 - 9); + HRESULT CFile::Parse(const Byte *p, UInt32 size) { - const UInt32 kHeadSize = 0xCC; + // CFile was initialized to default values: 0 in size variables and (IsCorrect == false) + const unsigned kHeadSize = 0xCC; if (size < kHeadSize) return S_FALSE; if (Get32(p) != 0x6D697368) // "mish" signature return S_FALSE; if (Get32(p + 4) != 1) // version return S_FALSE; - // UInt64 firstSectorNumber = Get64(p + 8); - UInt64 numSectors = Get64(p + 0x10); - - StartPos = Get64(p + 0x18); - - // UInt32 decompressedBufRequested = Get32(p + 0x20); // ??? - // UInt32 blocksDescriptor = Get32(p + 0x24); // number starting from -1? + + StartUnpackSector = Get64(p + 8); + NumUnpackSectors = Get64(p + 0x10); + StartPackPos = Get64(p + 0x18); + +#ifdef SHOW_DEBUG_INFO + /* the number of sectors that must be allocated. + == 0x208 for 256KB clusters + == 0x808 for 1MB clusters + == 0x1001 for 1MB clusters in some example + */ + const UInt32 decompressedBufRequested = Get32(p + 0x20); +#endif + + // Descriptor is file index. usually started from -1 + // in one dmg it was started from 0 + Descriptor = (Int32)Get32(p + 0x24); // char Reserved1[24]; Checksum.Parse(p + 0x40); - PRF(printf("\n\nChecksum Type = %2d", Checksum.Type)); - - UInt32 numBlocks = Get32(p + 0xC8); - if (numBlocks > ((UInt32)1 << 28)) - return S_FALSE; - - const UInt32 kRecordSize = 40; - if (numBlocks * kRecordSize + kHeadSize != size) + PRF(printf("\n" " Checksum Type = %2u" + "\n StartUnpackSector = %8x" + "\n NumUnpackSectors = %8x" + "\n StartPos = %8x" + "\n decompressedBufRequested=%8x" + "\n blocksDescriptor=%8x" + , (unsigned)Checksum.Type + , (unsigned)StartUnpackSector + , (unsigned)NumUnpackSectors + , (unsigned)StartPackPos + , (unsigned)decompressedBufRequested + , (unsigned)Descriptor + );) + + const UInt32 numBlocks = Get32(p + 0xC8); + const unsigned kRecordSize = 40; + if ((UInt64)numBlocks * kRecordSize + kHeadSize != size) return S_FALSE; - PackSize = 0; - Size = 0; Blocks.ClearAndReserve(numBlocks); FullFileChecksum = true; + /* IsCorrect = false; by default + So we return S_OK, if we can ignore some error in headers. + */ p += kHeadSize; UInt32 i; @@ -399,64 +704,104 @@ HRESULT CFile::Parse(const Byte *p, UInt32 size) { CBlock b; b.Type = Get32(p); - b.UnpPos = Get64(p + 0x08) << 9; - b.UnpSize = Get64(p + 0x10) << 9; + { + const UInt64 a = Get64(p + 0x08); + if (a >= kSectorNumber_LIMIT) + return S_OK; + b.UnpPos = a << 9; + } + UInt64 unpSize; + { + const UInt64 a = Get64(p + 0x10); + if (a >= kSectorNumber_LIMIT) + return S_OK; + unpSize = a << 9; + } + const UInt64 newSize = b.UnpPos + unpSize; + if (newSize >= ((UInt64)1 << 63)) + return S_OK; + b.PackPos = Get64(p + 0x18); b.PackSize = Get64(p + 0x20); - // b.PackPos can be 0 for some types. So we don't check it - if (!Blocks.IsEmpty()) - if (b.UnpPos != Blocks.Back().GetNextUnpPos()) - return S_FALSE; + if (b.UnpPos != Size) + return S_OK; - PRF(printf("\nType=%8x m[1]=%8x uPos=%8x uSize=%7x pPos=%8x pSize=%7x", - b.Type, Get32(p + 4), (UInt32)b.UnpPos, (UInt32)b.UnpSize, (UInt32)b.PackPos, (UInt32)b.PackSize)); + PRF(printf("\nType=%8x comment=%8x uPos=%8x uSize=%7x pPos=%8x pSize=%7x", + (unsigned)b.Type, Get32(p + 4), (unsigned)b.UnpPos, (unsigned)unpSize, (unsigned)b.PackPos, (unsigned)b.PackSize)); if (b.Type == METHOD_COMMENT) + { + // some files contain 2 comment block records: + // record[0] : Type=METHOD_COMMENT, comment_field = "+beg" + // record[num-2] : Type=METHOD_COMMENT, comment_field = "+end" + // we skip these useless records. continue; + } if (b.Type == METHOD_END) break; - PackSize += b.PackSize; - - if (b.UnpSize != 0) + + // we add only blocks that have non empty unpacked data: + if (unpSize != 0) { - if (b.Type == METHOD_ZERO_2) + const UInt64 k_max_pos = (UInt64)1 << 63; + if (b.PackPos >= k_max_pos || + b.PackSize >= k_max_pos - b.PackPos) + return S_OK; + + /* we don't count non-ZERO blocks here, because + ZERO blocks in dmg files are not limited by some cluster size. + note: COPY blocks also sometimes are fused to larger blocks. + */ + if (b.IsClusteredMethod()) + if (BlockSize_MAX < unpSize) + BlockSize_MAX = unpSize; + + PackSize += b.PackSize; + if (!b.NeedCrc()) FullFileChecksum = false; Blocks.AddInReserved(b); + Size = newSize; } } - + + PRF(printf("\n");) + if (i != numBlocks - 1) - return S_FALSE; - if (!Blocks.IsEmpty()) - Size = Blocks.Back().GetNextUnpPos(); - if (Size != (numSectors << 9)) - return S_FALSE; - + { + // return S_FALSE; + return S_OK; + } + + if ((Size >> 9) == NumUnpackSectors) + IsCorrect = true; return S_OK; } -static int FindKeyPair(const CXmlItem &item, const char *key, const char *nextTag) + +static const CXmlItem *FindKeyPair(const CXmlItem &item, const char *key, const char *nextTag) { for (unsigned i = 0; i + 1 < item.SubItems.Size(); i++) { const CXmlItem &si = item.SubItems[i]; - if (si.IsTagged("key") && si.GetSubString() == key && item.SubItems[i + 1].IsTagged(nextTag)) - return (int)(i + 1); + if (si.IsTagged("key") && si.GetSubString() == key) + { + const CXmlItem *si_1 = &item.SubItems[i + 1]; + if (si_1->IsTagged(nextTag)) + return si_1; + } } - return -1; + return NULL; } static const AString *GetStringFromKeyPair(const CXmlItem &item, const char *key, const char *nextTag) { - int index = FindKeyPair(item, key, nextTag); - if (index >= 0) - return item.SubItems[index].GetSubStringPtr(); + const CXmlItem *si_1 = FindKeyPair(item, key, nextTag); + if (si_1) + return si_1->GetSubStringPtr(); return NULL; } -static const unsigned HEADER_SIZE = 0x200; - static const Byte k_Signature[] = { 'k','o','l','y', 0, 0, 0, 4, 0, 0, 2, 0 }; static inline bool IsKoly(const Byte *p) @@ -485,144 +830,175 @@ HRESULT CHandler::ReadData(IInStream *stream, const CForkPair &pair, CByteBuffer } + bool CHandler::ParseBlob(const CByteBuffer &data) { - if (data.Size() < 12) + const unsigned kHeaderSize = 3 * 4; + if (data.Size() < kHeaderSize) return false; - const Byte *p = (const Byte *)data; - if (Get32(p) != 0xFADE0CC0) + const Byte * const p = (const Byte *)data; + if (Get32a(p) != 0xfade0cc0) // CSMAGIC_EMBEDDED_SIGNATURE return true; - const UInt32 size = Get32(p + 4); + const UInt32 size = Get32a(p + 4); if (size != data.Size()) return false; - const UInt32 num = Get32(p + 8); - if (num > (size - 12) / 8) + const UInt32 num = Get32a(p + 8); + if (num > (size - kHeaderSize) / 8) return false; - for (UInt32 i = 0; i < num; i++) + const UInt32 limit = num * 8 + kHeaderSize; + for (size_t i = kHeaderSize; i < limit; i += 8) { - // UInt32 type = Get32(p + i * 8 + 12); - UInt32 offset = Get32(p + i * 8 + 12 + 4); - if (size - offset < 8) + // type == 0 == CSSLOT_CODEDIRECTORY for CSMAGIC_CODEDIRECTORY item + // UInt32 type = Get32(p + i); + const UInt32 offset = Get32a(p + i + 4); + if (offset < limit || offset > size - 8) return false; - const Byte *p2 = (const Byte *)data + offset; + // offset is not aligned for 4 here !!! + const Byte * const p2 = p + offset; const UInt32 magic = Get32(p2); const UInt32 len = Get32(p2 + 4); if (size - offset < len || len < 8) return false; - #ifdef DMG_SHOW_RAW +#ifdef DMG_SHOW_RAW CExtraFile &extra = _extras.AddNew(); extra.Name = "_blob_"; extra.Data.CopyFrom(p2, len); - #endif +#endif - if (magic == 0xFADE0C02) + if (magic == 0xfade0c02) // CSMAGIC_CODEDIRECTORY { - #ifdef DMG_SHOW_RAW +#ifdef DMG_SHOW_RAW extra.Name += "codedir"; - #endif +#endif if (len < 11 * 4) return false; - UInt32 idOffset = Get32(p2 + 0x14); + const UInt32 idOffset = Get32(p2 + 5 * 4); if (idOffset >= len) return false; UInt32 len2 = len - idOffset; - if (len2 < (1 << 10)) - _name.SetFrom_CalcLen((const char *)(p2 + idOffset), len2); + const UInt32 kNameLenMax = 1 << 8; + if (len2 > kNameLenMax) + len2 = kNameLenMax; + _name.SetFrom_CalcLen((const char *)(p2 + idOffset), len2); + /* + // #define kSecCodeSignatureHashSHA1 1 + // #define kSecCodeSignatureHashSHA256 2 + const UInt32 hashOffset = Get32(p2 + 4 * 4); + const UInt32 nSpecialSlots = Get32(p2 + 6 * 4); + const UInt32 nCodeSlots = Get32(p2 + 7 * 4); + const unsigned hashSize = p2[36]; + const unsigned hashType = p2[37]; + // const unsigned unused = p2[38]; + const unsigned pageSize = p2[39]; + */ } - #ifdef DMG_SHOW_RAW - else if (magic == 0xFADE0C01) - extra.Name += "requirements"; - else if (magic == 0xFADE0B01) - extra.Name += "signed"; +#ifdef DMG_SHOW_RAW + else if (magic == 0xfade0c01) extra.Name += "requirements"; + else if (magic == 0xfade0b01) extra.Name += "signed"; else { char temp[16]; ConvertUInt32ToHex8Digits(magic, temp); extra.Name += temp; } - #endif +#endif } return true; } -HRESULT CHandler::Open2(IInStream *stream) + + +HRESULT CHandler::Open2(IInStream *stream, IArchiveOpenCallback *openArchiveCallback) { /* - usual dmg contains Koly Header at the end: - - rare case dmg contains Koly Header at the start. + - rare case old dmg contains Koly Header at the begin. */ - _dataStartOffset = 0; + // _dataStartOffset = 0; UInt64 fileSize; RINOK(InStream_GetPos_GetSize(stream, _startPos, fileSize)) - Byte buf[HEADER_SIZE]; + const unsigned HEADER_SIZE = 0x200; + UInt64 buf[HEADER_SIZE / sizeof(UInt64)]; RINOK(ReadStream_FALSE(stream, buf, HEADER_SIZE)) UInt64 headerPos; - bool startKolyMode = false; + bool front_Koly_Mode = false; - if (IsKoly(buf)) + /* + _dataForkChecksum.Offset == 0 for koly-at-the-end + _dataForkChecksum.Offset == 512 for koly-at-the-start + so we can use (_dataForkChecksum.Offset) to detect "koly-at-the-start" mode + */ + + if (IsKoly((const Byte *)(const void *)buf)) { // it can be normal koly-at-the-end or koly-at-the-start headerPos = _startPos; + /* if (_startPos <= (1 << 8)) { - // we want to support startKolyMode, even if there is + // we want to support front_Koly_Mode, even if there is // some data before dmg file, like 128 bytes MacBin header _dataStartOffset = HEADER_SIZE; - startKolyMode = true; + front_Koly_Mode = true; } + */ } else { - // we check only koly-at-the-end + /* we try to open in backward mode only for first attempt + when (_startPos == 0) */ + if (_startPos != 0) + return S_FALSE; headerPos = fileSize; if (headerPos < HEADER_SIZE) return S_FALSE; headerPos -= HEADER_SIZE; RINOK(InStream_SeekSet(stream, headerPos)) RINOK(ReadStream_FALSE(stream, buf, HEADER_SIZE)) - if (!IsKoly(buf)) + if (!IsKoly((const Byte *)(const void *)buf)) return S_FALSE; } - // UInt32 flags = Get32(buf + 12); - // UInt64 runningDataForkOffset = Get64(buf + 0x10); - - CForkPair dataForkPair, rsrcPair, xmlPair, blobPair; - - dataForkPair.Parse(buf + 0x18); - rsrcPair.Parse(buf + 0x28); - xmlPair.Parse(buf + 0xD8); - blobPair.Parse(buf + 0x128); - - // UInt32 segmentNumber = Get32(buf + 0x38); - // UInt32 segmentCount = Get32(buf + 0x3C); - // Byte segmentGUID[16]; - // CChecksum dataForkChecksum; - // dataForkChecksum.Parse(buf + 0x50); - + // UInt32 flags = Get32a((const Byte *)(const void *)buf + 12); + // _runningDataForkOffset = Get64a((const Byte *)(const void *)buf + 0x10); + _dataForkPair.Parse((const Byte *)(const void *)buf + 0x18); + rsrcPair.Parse((const Byte *)(const void *)buf + 0x28); + // _segmentNumber = Get32a(buf + 0x38); // 0 or 1 + // _segmentCount = Get32a(buf + 0x3C); // 0 (if not set) or 1 + memcpy(_segmentGUID, (const Byte *)(const void *)buf + 0x40, 16); + _dataForkChecksum.Parse((const Byte *)(const void *)buf + 0x50); + xmlPair.Parse((const Byte *)(const void *)buf + 0xD8); + // Byte resereved[] + blobPair.Parse((const Byte *)(const void *)buf + 0x128); + _masterChecksum.Parse((const Byte *)(const void *)buf + 0x160); + // UInt32 imageVariant = Get32a((const Byte *)(const void *)buf + 0x1E8); imageVariant = imageVariant; + _numSectors = Get64((const Byte *)(const void *)buf + 0x1EC); // it's not aligned for 8-bytes + // Byte resereved[12]; + + if (_dataForkPair.Offset == HEADER_SIZE + && headerPos + HEADER_SIZE < fileSize) + front_Koly_Mode = true; + + const UInt64 limit = front_Koly_Mode ? fileSize : headerPos; UInt64 top = 0; - UInt64 limit = startKolyMode ? fileSize : headerPos; - - if (!dataForkPair.UpdateTop(limit, top)) return S_FALSE; + if (!_dataForkPair.UpdateTop(limit, top)) return S_FALSE; if (!xmlPair.UpdateTop(limit, top)) return S_FALSE; if (!rsrcPair.UpdateTop(limit, top)) return S_FALSE; /* Some old dmg files contain garbage data in blobPair field. So we need to ignore such garbage case; And we still need to detect offset of start of archive for "parser" mode. */ + const bool useBlob = blobPair.UpdateTop(limit, top); - bool useBlob = blobPair.UpdateTop(limit, top); - - - if (startKolyMode) + if (front_Koly_Mode) _phySize = top; else { @@ -637,71 +1013,78 @@ HRESULT CHandler::Open2(IInStream *stream) - archive with offset. So we try to read XML with absolute offset to select from these two ways. */ - CForkPair xmlPair2 = xmlPair; - const char *sz = " len) - xmlPair2.Len = len; - CByteBuffer buf2; - if (xmlPair2.Len < len + CForkPair xmlPair2 = xmlPair; + const char *sz = " len) + xmlPair2.Len = len; + CByteBuffer buf2; + if (xmlPair2.Len < len || ReadData(stream, xmlPair2, buf2) != S_OK || memcmp(buf2, sz, len) != 0) - { - // if absolute offset is not OK, probably it's archive with offset - _startPos = headerPos - top; - _phySize = top + HEADER_SIZE; - } + { + // if absolute offset is not OK, probably it's archive with offset + _startPos = headerPos - top; + _phySize = top + HEADER_SIZE; + } } } - // Byte reserved[0x78] - - if (useBlob && blobPair.Len != 0) + if (useBlob + && blobPair.Len != 0 + && blobPair.Len <= (1u << 24)) // we don't want parsing of big blobs { - #ifdef DMG_SHOW_RAW +#ifdef DMG_SHOW_RAW CExtraFile &extra = _extras.AddNew(); extra.Name = "_blob.bin"; CByteBuffer &blobBuf = extra.Data; - #else +#else CByteBuffer blobBuf; - #endif +#endif RINOK(ReadData(stream, blobPair, blobBuf)) if (!ParseBlob(blobBuf)) _headersError = true; } - CChecksum masterChecksum; - masterChecksum.Parse(buf + 0x160); - - // UInt32 imageVariant = Get32(buf + 0x1E8); - // UInt64 numSectors = Get64(buf + 0x1EC); - // Byte reserved[0x12] + UInt64 openTotal = 0; + UInt64 openCur = 0; + if (_dataForkChecksum.IsCrc32()) + openTotal = _dataForkPair.Len; const UInt32 RSRC_HEAD_SIZE = 0x100; - // We don't know the size of the field "offset" in rsrc. - // We suppose that it uses 24 bits. So we use Rsrc, only if the rsrcLen < (1 << 24). - bool useRsrc = (rsrcPair.Len > RSRC_HEAD_SIZE && rsrcPair.Len < ((UInt32)1 << 24)); - // useRsrc = false; - - if (useRsrc) + /* we have 2 ways to read files and blocks metadata: + via Xml or via Rsrc. + But some images have no Rsrc fork. + Is it possible that there is no Xml? + Rsrc method will not work for big files. */ + // v23.02: we use xml mode by default + // if (rsrcPair.Len >= RSRC_HEAD_SIZE && rsrcPair.Len <= ((UInt32)1 << 24)) // for debug + if (xmlPair.Len == 0) { - #ifdef DMG_SHOW_RAW + // We don't know the size of the field "offset" in Rsrc. + // We suppose that it uses 24 bits. So we use Rsrc, only if the rsrcPair.Len <= (1 << 24). + const bool canUseRsrc = (rsrcPair.Len >= RSRC_HEAD_SIZE && rsrcPair.Len <= ((UInt32)1 << 24)); + if (!canUseRsrc) + return S_FALSE; + + _rsrcMode_wasUsed = true; +#ifdef DMG_SHOW_RAW CExtraFile &extra = _extras.AddNew(); extra.Name = "rsrc.bin"; CByteBuffer &rsrcBuf = extra.Data; - #else +#else CByteBuffer rsrcBuf; - #endif +#endif RINOK(ReadData(stream, rsrcPair, rsrcBuf)) const Byte *p = rsrcBuf; - UInt32 headSize = Get32(p + 0); - UInt32 footerOffset = Get32(p + 4); - UInt32 mainDataSize = Get32(p + 8); - UInt32 footerSize = Get32(p + 12); + const UInt32 headSize = Get32a(p + 0); + const UInt32 footerOffset = Get32a(p + 4); + const UInt32 mainDataSize = Get32a(p + 8); + const UInt32 footerSize = Get32a(p + 12); if (headSize != RSRC_HEAD_SIZE || footerOffset >= rsrcPair.Len || mainDataSize >= rsrcPair.Len @@ -709,59 +1092,59 @@ HRESULT CHandler::Open2(IInStream *stream) || footerOffset != headSize + mainDataSize) return S_FALSE; - const UInt32 footerEnd = footerOffset + footerSize; - if (footerEnd != rsrcPair.Len) { - // there is rare case dmg example, where there are 4 additional bytes - UInt64 rem = rsrcPair.Len - footerOffset; - if (rem < footerSize + const UInt32 footerEnd = footerOffset + footerSize; + if (footerEnd != rsrcPair.Len) + { + // there is rare case dmg example, where there are 4 additional bytes + const UInt64 rem = rsrcPair.Len - footerOffset; + if (rem < footerSize || rem - footerSize != 4 || Get32(p + footerEnd) != 0) - return S_FALSE; + return S_FALSE; + } } - if (footerSize < 16) + if (footerSize < 0x1e) return S_FALSE; if (memcmp(p, p + footerOffset, 16) != 0) return S_FALSE; p += footerOffset; - if ((UInt32)Get16(p + 0x18) != 0x1C) + if ((UInt32)Get16(p + 0x18) != 0x1c) return S_FALSE; - const UInt32 namesOffset = Get16(p + 0x1A); + const UInt32 namesOffset = Get16(p + 0x1a); if (namesOffset > footerSize) return S_FALSE; - UInt32 numItems = (UInt32)Get16(p + 0x1C) + 1; - if (numItems * 8 + 0x1E > namesOffset) + const UInt32 numItems = (UInt32)Get16(p + 0x1c) + 1; + if (numItems * 8 + 0x1e > namesOffset) return S_FALSE; for (UInt32 i = 0; i < numItems; i++) { - const Byte *p2 = p + 0x1E + i * 8; - + const Byte *p2 = p + 0x1e + (size_t)i * 8; const UInt32 typeId = Get32(p2); - - #ifndef DMG_SHOW_RAW - if (typeId != 0x626C6B78) // blkx + const UInt32 k_typeId_blkx = 0x626c6b78; // blkx +#ifndef DMG_SHOW_RAW + if (typeId != k_typeId_blkx) continue; - #endif - +#endif const UInt32 numFiles = (UInt32)Get16(p2 + 4) + 1; const UInt32 offs = Get16(p2 + 6); - if (0x1C + offs + 12 * numFiles > namesOffset) + if (0x1c + offs + 12 * numFiles > namesOffset) return S_FALSE; for (UInt32 k = 0; k < numFiles; k++) { - const Byte *p3 = p + 0x1C + offs + k * 12; + const Byte *p3 = p + 0x1c + offs + k * 12; // UInt32 id = Get16(p3); const UInt32 namePos = Get16(p3 + 2); - // Byte attributes = p3[4]; // = 0x50 for blkx + // Byte attributes = p3[4]; // = 0x50 for blkx #define (ATTRIBUTE_HDIUTIL) // we don't know how many bits we can use. So we use 24 bits only UInt32 blockOffset = Get32(p3 + 4); - blockOffset &= (((UInt32)1 << 24) - 1); + blockOffset &= ((UInt32)1 << 24) - 1; // UInt32 unknown2 = Get32(p3 + 8); // ??? if (blockOffset + 4 >= mainDataSize) return S_FALSE; @@ -772,38 +1155,38 @@ HRESULT CHandler::Open2(IInStream *stream) AString name; - if (namePos != 0xFFFF) + if (namePos != 0xffff) { - UInt32 namesBlockSize = footerSize - namesOffset; + const UInt32 namesBlockSize = footerSize - namesOffset; if (namePos >= namesBlockSize) return S_FALSE; const Byte *namePtr = p + namesOffset + namePos; - UInt32 nameLen = *namePtr; + const UInt32 nameLen = *namePtr; if (namesBlockSize - namePos <= nameLen) return S_FALSE; for (UInt32 r = 1; r <= nameLen; r++) { - Byte c = namePtr[r]; + const Byte c = namePtr[r]; if (c < 0x20 || c >= 0x80) break; name += (char)c; } } - if (typeId == 0x626C6B78) // blkx + if (typeId == k_typeId_blkx) { CFile &file = _files.AddNew(); file.Name = name; RINOK(file.Parse(pBlock + 4, blockSize)) + if (!file.IsCorrect) + _headersError = true; } - #ifdef DMG_SHOW_RAW +#ifdef DMG_SHOW_RAW { AString name2; - name2.Add_UInt32(i); name2 += '_'; - { char temp[4 + 1] = { 0 }; memcpy(temp, p2, 4); @@ -812,69 +1195,80 @@ HRESULT CHandler::Open2(IInStream *stream) name2.Trim(); name2 += '_'; name2.Add_UInt32(k); - if (!name.IsEmpty()) { name2 += '_'; name2 += name; } - - CExtraFile &extra = _extras.AddNew(); - extra.Name = name2; - extra.Data.CopyFrom(pBlock + 4, blockSize); + CExtraFile &extra2 = _extras.AddNew(); + extra2.Name = name2; + extra2.Data.CopyFrom(pBlock + 4, blockSize); } - #endif +#endif } } } else { - if (xmlPair.Len >= kXmlSizeMax || xmlPair.Len == 0) + if (xmlPair.Len > k_XmlSize_MAX) return S_FALSE; - size_t size = (size_t)xmlPair.Len; - if (size != xmlPair.Len) - return S_FALSE; - + // if (!canUseXml) return S_FALSE; + const size_t size = (size_t)xmlPair.Len; + // if (size + 1 <= xmlPair.Len) return S_FALSE; // optional check RINOK(InStream_SeekSet(stream, _startPos + xmlPair.Offset)) - CXml xml; { - CObjArray xmlStr(size + 1); - RINOK(ReadStream_FALSE(stream, xmlStr, size)) + openTotal += size; + if (openArchiveCallback) + { + RINOK(openArchiveCallback->SetTotal(NULL, &openTotal)) + } + size_t pos = 0; + CAlignedBuffer1 xmlStr(size + 1); + for (;;) + { + const size_t k_OpenStep = 1 << 24; + const size_t cur = MyMin(k_OpenStep, size - pos); + RINOK(ReadStream_FALSE(stream, xmlStr + pos, cur)) + pos += cur; + openCur += cur; + if (pos == size) + break; + if (openArchiveCallback) + { + RINOK(openArchiveCallback->SetCompleted(NULL, &openCur)) + } + } xmlStr[size] = 0; - // if (strlen(xmlStr) != size) return S_FALSE; - if (!xml.Parse(xmlStr)) + // if (strlen((const char *)(const void *)(const Byte *)xmlStr) != size) return S_FALSE; + if (!xml.Parse((char *)(void *)(Byte *)xmlStr)) return S_FALSE; - #ifdef DMG_SHOW_RAW +#ifdef DMG_SHOW_RAW CExtraFile &extra = _extras.AddNew(); extra.Name = "a.xml"; - extra.Data.CopyFrom((const Byte *)(const char *)xmlStr, size); - #endif + extra.Data.CopyFrom(xmlStr, size); +#endif } if (xml.Root.Name != "plist") return S_FALSE; - int dictIndex = xml.Root.FindSubTag("dict"); - if (dictIndex < 0) + const CXmlItem *dictItem = xml.Root.FindSubTag_GetPtr("dict"); + if (!dictItem) return S_FALSE; - const CXmlItem &dictItem = xml.Root.SubItems[dictIndex]; - int rfDictIndex = FindKeyPair(dictItem, "resource-fork", "dict"); - if (rfDictIndex < 0) + const CXmlItem *rfDictItem = FindKeyPair(*dictItem, "resource-fork", "dict"); + if (!rfDictItem) return S_FALSE; - const CXmlItem &rfDictItem = dictItem.SubItems[rfDictIndex]; - int arrIndex = FindKeyPair(rfDictItem, "blkx", "array"); - if (arrIndex < 0) + const CXmlItem *arrItem = FindKeyPair(*rfDictItem, "blkx", "array"); + if (!arrItem) return S_FALSE; - const CXmlItem &arrItem = rfDictItem.SubItems[arrIndex]; - - FOR_VECTOR (i, arrItem.SubItems) + FOR_VECTOR (i, arrItem->SubItems) { - const CXmlItem &item = arrItem.SubItems[i]; + const CXmlItem &item = arrItem->SubItems[i]; if (!item.IsTagged("dict")) continue; @@ -893,25 +1287,38 @@ HRESULT CHandler::Open2(IInStream *stream) destLen = (unsigned)(endPtr - (const Byte *)rawBuf); } - #ifdef DMG_SHOW_RAW +#ifdef DMG_SHOW_RAW CExtraFile &extra = _extras.AddNew(); extra.Name.Add_UInt32(_files.Size()); extra.Data.CopyFrom(rawBuf, destLen); - #endif +#endif } CFile &file = _files.AddNew(); { + /* xml code removes front space for such string: + (Apple_Free : 3) + maybe we shoud fix xml code and return full string with space. + */ const AString *name = GetStringFromKeyPair(item, "Name", "string"); if (!name || name->IsEmpty()) name = GetStringFromKeyPair(item, "CFName", "string"); if (name) file.Name = *name; } + /* + { + const AString *s = GetStringFromKeyPair(item, "ID", "string"); + if (s) + file.Id = *s; + } + */ RINOK(file.Parse(rawBuf, destLen)) + if (!file.IsCorrect) + _headersError = true; } } - if (masterChecksum.IsCrc32()) + if (_masterChecksum.IsCrc32()) { UInt32 crc = CRC_INIT_VAL; unsigned i; @@ -920,54 +1327,127 @@ HRESULT CHandler::Open2(IInStream *stream) const CChecksum &cs = _files[i].Checksum; if ((cs.NumBits & 0x7) != 0) break; - UInt32 len = cs.NumBits >> 3; + const UInt32 len = cs.NumBits >> 3; if (len > kChecksumSize_Max) break; crc = CrcUpdate(crc, cs.Data, (size_t)len); } if (i == _files.Size()) - _masterCrcError = (CRC_GET_DIGEST(crc) != masterChecksum.GetCrc32()); + _masterCrcError = (CRC_GET_DIGEST(crc) != _masterChecksum.GetCrc32()); } - return S_OK; -} - + { + UInt64 sec = 0; + FOR_VECTOR (i, _files) + { + const CFile &file = _files[i]; + /* + if (file.Descriptor != (Int32)i - 1) + _headersError = true; + */ + if (file.StartUnpackSector != sec) + _headersError = true; + if (file.NumUnpackSectors >= kSectorNumber_LIMIT) + _headersError = true; + sec += file.NumUnpackSectors; + if (sec >= kSectorNumber_LIMIT) + _headersError = true; + } + if (sec != _numSectors) + _headersError = true; + } + + // data checksum calculation can be slow for big dmg file + if (_dataForkChecksum.IsCrc32()) + { + UInt64 endPos; + if (!_dataForkPair.GetEndPos(endPos) + || _dataForkPair.Offset >= ((UInt64)1 << 63)) + _headersError = true; + else + { + const UInt64 seekPos = _startPos + _dataForkPair.Offset; + if (seekPos > fileSize + || endPos > fileSize - _startPos) + { + _headersError = true; + // kpv_ErrorFlags_UnexpectedEnd + } + else + { + const size_t kBufSize = 1 << 15; + CAlignedBuffer1 buf2(kBufSize); + RINOK(InStream_SeekSet(stream, seekPos)) + if (openArchiveCallback) + { + RINOK(openArchiveCallback->SetTotal(NULL, &openTotal)) + } + UInt32 crc = CRC_INIT_VAL; + UInt64 pos = 0; + for (;;) + { + const UInt64 rem = _dataForkPair.Len - pos; + size_t cur = kBufSize; + if (cur > rem) + cur = (UInt32)rem; + if (cur == 0) + break; + RINOK(ReadStream_FALSE(stream, buf2, cur)) + crc = CrcUpdate(crc, buf2, cur); + pos += cur; + openCur += cur; + if ((pos & ((1 << 24) - 1)) == 0 && openArchiveCallback) + { + RINOK(openArchiveCallback->SetCompleted(NULL, &openCur)) + } + } + if (_dataForkChecksum.GetCrc32() != CRC_GET_DIGEST(crc)) + _dataForkError = true; + } + } + } + + return S_OK; +} + + + Z7_COM7F_IMF(CHandler::Open(IInStream *stream, const UInt64 * /* maxCheckStartPosition */, - IArchiveOpenCallback * /* openArchiveCallback */)) + IArchiveOpenCallback *openArchiveCallback)) { COM_TRY_BEGIN - { - Close(); - if (Open2(stream) != S_OK) - return S_FALSE; - _inStream = stream; - } + Close(); + RINOK(Open2(stream, openArchiveCallback)) + _inStream = stream; return S_OK; COM_TRY_END } Z7_COM7F_IMF(CHandler::Close()) { - _phySize = 0; - _inStream.Release(); - _files.Clear(); _masterCrcError = false; _headersError = false; + _dataForkError = false; + _rsrcMode_wasUsed = false; + _phySize = 0; + _startPos = 0; _name.Empty(); - #ifdef DMG_SHOW_RAW + _inStream.Release(); + _files.Clear(); +#ifdef DMG_SHOW_RAW _extras.Clear(); - #endif +#endif return S_OK; } Z7_COM7F_IMF(CHandler::GetNumberOfItems(UInt32 *numItems)) { *numItems = _files.Size() - #ifdef DMG_SHOW_RAW +#ifdef DMG_SHOW_RAW + _extras.Size() - #endif - ; +#endif + ; return S_OK; } @@ -980,7 +1460,7 @@ Z7_COM7F_IMF(CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val COM_TRY_BEGIN NWindows::NCOM::CPropVariant prop; - #ifdef DMG_SHOW_RAW +#ifdef DMG_SHOW_RAW if (index >= _files.Size()) { const CExtraFile &extra = _extras[index - _files.Size()]; @@ -996,7 +1476,7 @@ Z7_COM7F_IMF(CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val } } else - #endif +#endif { const CFile &item = _files[index]; switch (propID) @@ -1009,21 +1489,44 @@ Z7_COM7F_IMF(CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val prop = item.Checksum.GetCrc32(); break; } + case kpidChecksum: + { + AString s; + item.Checksum.Print(s); + if (!s.IsEmpty()) + prop = s; + break; + } /* case kpidOffset: { - prop = item.StartPos; + prop = item.StartPackPos; break; } */ + case kpidNumBlocks: + prop = (UInt32)item.Blocks.Size(); + break; + case kpidClusterSize: + prop = item.BlockSize_MAX; + break; + case kpidMethod: { + AString s; + if (!item.IsCorrect) + s.Add_OptSpaced("CORRUPTED"); CMethods m; m.Update(item); - AString s; - m.GetString(s); + m.AddToString(s); + { + AString s2; + item.Checksum.PrintType(s2); + if (!s2.IsEmpty()) + s.Add_OptSpaced(s2); + } if (!s.IsEmpty()) prop = s; break; @@ -1031,6 +1534,9 @@ Z7_COM7F_IMF(CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val case kpidPath: { +#ifdef Z7_DMG_SINGLE_FILE_MODE + prop = "a.img"; +#else UString name; name.Add_UInt32(index); unsigned num = 10; @@ -1059,18 +1565,9 @@ Z7_COM7F_IMF(CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val subName.Trim(); if (!subName.IsEmpty()) { - for (unsigned n = 0; n < kNumAppleNames; n++) - { - const CAppleName &appleName = k_Names[n]; - if (appleName.Ext) - { - if (subName == appleName.AppleName) - { - subName = appleName.Ext; - break; - } - } - } + const char *ext = Find_Apple_FS_Ext(subName); + if (ext) + subName = ext; UString name2; ConvertUTF8ToUnicode(subName, name2); name.Add_Dot(); @@ -1085,6 +1582,8 @@ Z7_COM7F_IMF(CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val name += name2; } prop = name; +#endif + break; } @@ -1095,6 +1594,24 @@ Z7_COM7F_IMF(CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val prop = name; break; } + case kpidId: + { + prop.Set_Int32((Int32)item.Descriptor); + /* + if (!item.Id.IsEmpty()) + { + UString s; + ConvertUTF8ToUnicode(item.Id, s); + prop = s; + } + */ + break; + } +#ifdef Z7_DMG_SINGLE_FILE_MODE + case kpidPosition: + prop = item.StartUnpackSector << 9; + break; +#endif } } prop.Detach(value); @@ -1103,21 +1620,11 @@ Z7_COM7F_IMF(CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val } -Z7_CLASS_IMP_NOQIB_1( - CAdcDecoder - , ICompressCoder -) +class CAdcDecoder +{ CLzOutWindow m_OutWindowStream; CInBuffer m_InStream; - /* - void ReleaseStreams() - { - m_OutWindowStream.ReleaseStream(); - m_InStream.ReleaseStream(); - } - */ - class CCoderReleaser Z7_final { CAdcDecoder *m_Coder; @@ -1128,22 +1635,28 @@ Z7_CLASS_IMP_NOQIB_1( { if (NeedFlush) m_Coder->m_OutWindowStream.Flush(); - // m_Coder->ReleaseStreams(); } }; friend class CCoderReleaser; public: - HRESULT CodeReal(ISequentialInStream *inStream, - ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, - ICompressProgressInfo *progress); + HRESULT Code(ISequentialInStream * const inStream, + ISequentialOutStream *outStream, + const UInt64 * const inSize, + const UInt64 * const outSize, + ICompressProgressInfo * const progress); }; -HRESULT CAdcDecoder::CodeReal(ISequentialInStream *inStream, - ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, - ICompressProgressInfo *progress) + +HRESULT CAdcDecoder::Code(ISequentialInStream * const inStream, + ISequentialOutStream *outStream, + const UInt64 * const inSize, + const UInt64 * const outSizePtr, + ICompressProgressInfo * const progress) { - if (!m_OutWindowStream.Create(1 << 18)) + try { + + if (!m_OutWindowStream.Create(1 << 18)) // at least (1 << 16) is required here return E_OUTOFMEMORY; if (!m_InStream.Create(1 << 18)) return E_OUTOFMEMORY; @@ -1155,73 +1668,72 @@ HRESULT CAdcDecoder::CodeReal(ISequentialInStream *inStream, CCoderReleaser coderReleaser(this); - const UInt32 kStep = (1 << 20); + const UInt32 kStep = 1 << 22; UInt64 nextLimit = kStep; - + const UInt64 outSize = *outSizePtr; UInt64 pos = 0; - while (pos < *outSize) + /* match sequences and literal sequences do not cross 64KB range + in some dmg archive examples. But is it so for any Adc stream? */ + + while (pos < outSize) { - if (pos > nextLimit && progress) + if (pos >= nextLimit && progress) { - UInt64 packSize = m_InStream.GetProcessedSize(); - RINOK(progress->SetRatioInfo(&packSize, &pos)) nextLimit += kStep; + const UInt64 packSize = m_InStream.GetProcessedSize(); + RINOK(progress->SetRatioInfo(&packSize, &pos)) } Byte b; if (!m_InStream.ReadByte(b)) return S_FALSE; - UInt64 rem = *outSize - pos; + const UInt64 rem = outSize - pos; if (b & 0x80) { - unsigned num = (b & 0x7F) + 1; + unsigned num = (unsigned)b - 0x80 + 1; if (num > rem) return S_FALSE; - for (unsigned i = 0; i < num; i++) + pos += num; + do { if (!m_InStream.ReadByte(b)) return S_FALSE; m_OutWindowStream.PutByte(b); } - pos += num; + while (--num); continue; } Byte b1; if (!m_InStream.ReadByte(b1)) return S_FALSE; - UInt32 len, distance; + UInt32 len, dist; if (b & 0x40) { - len = ((UInt32)b & 0x3F) + 4; + len = (UInt32)b - 0x40 + 4; Byte b2; if (!m_InStream.ReadByte(b2)) return S_FALSE; - distance = ((UInt32)b1 << 8) + b2; + dist = ((UInt32)b1 << 8) + b2; } else { - b &= 0x3F; len = ((UInt32)b >> 2) + 3; - distance = (((UInt32)b & 3) << 8) + b1; + dist = (((UInt32)b & 3) << 8) + b1; } - if (distance >= pos || len > rem) + if (/* dist >= pos || */ len > rem) + return S_FALSE; + if (!m_OutWindowStream.CopyBlock(dist, len)) return S_FALSE; - m_OutWindowStream.CopyBlock(distance, len); pos += len; } if (*inSize != m_InStream.GetProcessedSize()) return S_FALSE; coderReleaser.NeedFlush = false; return m_OutWindowStream.Flush(); -} -Z7_COM7F_IMF(CAdcDecoder::Code(ISequentialInStream *inStream, - ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, - ICompressProgressInfo *progress)) -{ - try { return CodeReal(inStream, outStream, inSize, outSize, progress);} + } catch(const CInBufferException &e) { return e.ErrorCode; } catch(const CLzOutWindowException &e) { return e.ErrorCode; } catch(...) { return S_FALSE; } @@ -1229,8 +1741,53 @@ Z7_COM7F_IMF(CAdcDecoder::Code(ISequentialInStream *inStream, +struct CDecoders +{ + CMyComPtr2 zlib; + CMyComPtr2 bzip2; + CMyComPtr2 lzfse; + CMyUniquePtr xz; + CMyUniquePtr adc; + + HRESULT Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, + const CBlock &block, const UInt64 *unpSize, ICompressProgressInfo *progress); +}; - +HRESULT CDecoders::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, + const CBlock &block, const UInt64 *unpSize, ICompressProgressInfo *progress) +{ + HRESULT hres; + UInt64 processed; + switch (block.Type) + { + case METHOD_ADC: + adc.Create_if_Empty(); + return adc->Code(inStream, outStream, &block.PackSize, unpSize, progress); + case METHOD_LZFSE: + lzfse.Create_if_Empty(); + return lzfse.Interface()->Code(inStream, outStream, &block.PackSize, unpSize, progress); + case METHOD_ZLIB: + zlib.Create_if_Empty(); + hres = zlib.Interface()->Code(inStream, outStream, NULL, unpSize, progress); + processed = zlib->GetInputProcessedSize(); + break; + case METHOD_BZIP2: + bzip2.Create_if_Empty(); + hres = bzip2.Interface()->Code(inStream, outStream, NULL, unpSize, progress); + processed = bzip2->GetInputProcessedSize(); + break; + case METHOD_XZ: + xz.Create_if_Empty(); + hres = xz->Decode(inStream, outStream, unpSize, true, progress); + processed = xz->Stat.InSize; + break; + default: + return E_NOTIMPL; + } + if (hres == S_OK && processed != block.PackSize) + hres = S_FALSE; + return hres; +} Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, @@ -1239,7 +1796,11 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, COM_TRY_BEGIN const bool allFilesMode = (numItems == (UInt32)(Int32)-1); if (allFilesMode) - numItems = _files.Size(); + numItems = _files.Size() +#ifdef DMG_SHOW_RAW + + _extras.Size() +#endif + ; if (numItems == 0) return S_OK; UInt64 totalSize = 0; @@ -1247,55 +1808,46 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, for (i = 0; i < numItems; i++) { - UInt32 index = (allFilesMode ? i : indices[i]); - #ifdef DMG_SHOW_RAW + const UInt32 index = allFilesMode ? i : indices[i]; +#ifdef DMG_SHOW_RAW if (index >= _files.Size()) totalSize += _extras[index - _files.Size()].Data.Size(); else - #endif +#endif totalSize += _files[index].Size; } - extractCallback->SetTotal(totalSize); + RINOK(extractCallback->SetTotal(totalSize)) - UInt64 currentPackTotal = 0; - UInt64 currentUnpTotal = 0; - UInt64 currentPackSize = 0; - UInt64 currentUnpSize = 0; - - const UInt32 kZeroBufSize = (1 << 14); - CByteBuffer zeroBuf(kZeroBufSize); + const size_t kZeroBufSize = 1 << 14; + CAlignedBuffer1 zeroBuf(kZeroBufSize); memset(zeroBuf, 0, kZeroBufSize); - NCompress::CCopyCoder *copyCoderSpec = new NCompress::CCopyCoder(); - CMyComPtr copyCoder = copyCoderSpec; - - NCompress::NBZip2::CDecoder *bzip2CoderSpec = new NCompress::NBZip2::CDecoder(); - CMyComPtr bzip2Coder = bzip2CoderSpec; - - NCompress::NZlib::CDecoder *zlibCoderSpec = new NCompress::NZlib::CDecoder(); - CMyComPtr zlibCoder = zlibCoderSpec; - - CAdcDecoder *adcCoderSpec = new CAdcDecoder(); - CMyComPtr adcCoder = adcCoderSpec; - - NCompress::NLzfse::CDecoder *lzfseCoderSpec = new NCompress::NLzfse::CDecoder(); - CMyComPtr lzfseCoder = lzfseCoderSpec; - - CLocalProgress *lps = new CLocalProgress; - CMyComPtr progress = lps; + CDecoders decoders; + CMyComPtr2_Create copyCoder; + CMyComPtr2_Create lps; lps->Init(extractCallback, false); + CMyComPtr2_Create inStream; + inStream->SetStream(_inStream); - CLimitedSequentialInStream *streamSpec = new CLimitedSequentialInStream; - CMyComPtr inStream(streamSpec); - streamSpec->SetStream(_inStream); + UInt64 total_PackSize = 0; + UInt64 total_UnpackSize = 0; + UInt64 cur_PackSize = 0; + UInt64 cur_UnpackSize = 0; - for (i = 0; i < numItems; i++, currentPackTotal += currentPackSize, currentUnpTotal += currentUnpSize) + for (i = 0;; i++, + total_PackSize += cur_PackSize, + total_UnpackSize += cur_UnpackSize) { - lps->InSize = currentPackTotal; - lps->OutSize = currentUnpTotal; - currentPackSize = 0; - currentUnpSize = 0; + lps->InSize = total_PackSize; + lps->OutSize = total_UnpackSize; + cur_PackSize = 0; + cur_UnpackSize = 0; RINOK(lps->SetCur()) + if (i >= numItems) + return S_OK; + + Int32 opRes = NExtract::NOperationResult::kOK; + { CMyComPtr realOutStream; const Int32 askMode = testMode ? NExtract::NAskMode::kTest : @@ -1307,227 +1859,209 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, continue; RINOK(extractCallback->PrepareOperation(askMode)) - - COutStreamWithCRC *outCrcStreamSpec = new COutStreamWithCRC; - CMyComPtr outCrcStream = outCrcStreamSpec; - outCrcStreamSpec->SetStream(realOutStream); - bool needCrc = false; - outCrcStreamSpec->Init(needCrc); - - CLimitedSequentialOutStream *outStreamSpec = new CLimitedSequentialOutStream; - CMyComPtr outStream(outStreamSpec); - outStreamSpec->SetStream(outCrcStream); - - realOutStream.Release(); - - Int32 opRes = NExtract::NOperationResult::kOK; - #ifdef DMG_SHOW_RAW +#ifdef DMG_SHOW_RAW if (index >= _files.Size()) { const CByteBuffer &buf = _extras[index - _files.Size()].Data; - outStreamSpec->Init(buf.Size()); - RINOK(WriteStream(outStream, buf, buf.Size())); - currentPackSize = currentUnpSize = buf.Size(); + if (realOutStream) + RINOK(WriteStream(realOutStream, buf, buf.Size())) + cur_PackSize = cur_UnpackSize = buf.Size(); } else - #endif +#endif { const CFile &item = _files[index]; - currentPackSize = item.PackSize; - currentUnpSize = item.Size; + cur_PackSize = item.PackSize; + cur_UnpackSize = item.Size; - needCrc = item.Checksum.IsCrc32(); - - UInt64 unpPos = 0; - UInt64 packPos = 0; + if (!item.IsCorrect) + opRes = NExtract::NOperationResult::kHeadersError; + else { - FOR_VECTOR (j, item.Blocks) + CMyComPtr2_Create outCrcStream; + outCrcStream->SetStream(realOutStream); + // realOutStream.Release(); + const bool needCrc = item.Checksum.IsCrc32(); + outCrcStream->Init(needCrc); + + CMyComPtr2_Create outStream; + outStream->SetStream(outCrcStream); + + UInt64 unpPos = 0; + UInt64 packPos = 0; + + FOR_VECTOR (blockIndex, item.Blocks) { - lps->InSize = currentPackTotal + packPos; - lps->OutSize = currentUnpTotal + unpPos; + lps->InSize = total_PackSize + packPos; + lps->OutSize = total_UnpackSize + unpPos; RINOK(lps->SetCur()) - const CBlock &block = item.Blocks[j]; - if (!block.ThereAreDataInBlock()) - continue; + const CBlock &block = item.Blocks[blockIndex]; + // if (!block.ThereAreDataInBlock()) continue; packPos += block.PackSize; if (block.UnpPos != unpPos) { - opRes = NExtract::NOperationResult::kDataError; + opRes = NExtract::NOperationResult::kHeadersError; break; } - RINOK(InStream_SeekSet(_inStream, _startPos + _dataStartOffset + item.StartPos + block.PackPos)) - streamSpec->Init(block.PackSize); - bool realMethod = true; - outStreamSpec->Init(block.UnpSize); - HRESULT res = S_OK; + RINOK(InStream_SeekSet(_inStream, _startPos + _dataForkPair.Offset + item.StartPackPos + block.PackPos)) + inStream->Init(block.PackSize); - outCrcStreamSpec->EnableCalc(needCrc); + const UInt64 unpSize = item.GetUnpackSize_of_Block(blockIndex); - switch (block.Type) - { - case METHOD_ZERO_0: - case METHOD_ZERO_2: - realMethod = false; - if (block.PackSize != 0) - opRes = NExtract::NOperationResult::kUnsupportedMethod; - outCrcStreamSpec->EnableCalc(block.Type == METHOD_ZERO_0); - break; - - case METHOD_COPY: - if (block.UnpSize != block.PackSize) - { - opRes = NExtract::NOperationResult::kUnsupportedMethod; - break; - } - res = copyCoder->Code(inStream, outStream, NULL, NULL, progress); - break; - - case METHOD_ADC: - { - res = adcCoder->Code(inStream, outStream, &block.PackSize, &block.UnpSize, progress); - break; - } - - case METHOD_ZLIB: - { - res = zlibCoder->Code(inStream, outStream, NULL, NULL, progress); - if (res == S_OK) - if (zlibCoderSpec->GetInputProcessedSize() != block.PackSize) - opRes = NExtract::NOperationResult::kDataError; - break; - } + outStream->Init(unpSize); + HRESULT res = S_OK; - case METHOD_BZIP2: - { - res = bzip2Coder->Code(inStream, outStream, NULL, NULL, progress); - if (res == S_OK) - if (bzip2CoderSpec->GetInputProcessedSize() != block.PackSize) - opRes = NExtract::NOperationResult::kDataError; - break; - } + outCrcStream->EnableCalc(needCrc && block.NeedCrc()); - case METHOD_LZFSE: - { - res = lzfseCoder->Code(inStream, outStream, &block.PackSize, &block.UnpSize, progress); - break; - } - - default: + if (block.IsZeroMethod()) + { + if (block.PackSize != 0) opRes = NExtract::NOperationResult::kUnsupportedMethod; - break; } + else if (block.Type == METHOD_COPY) + { + if (unpSize != block.PackSize) + opRes = NExtract::NOperationResult::kUnsupportedMethod; + else + res = copyCoder.Interface()->Code(inStream, outStream, NULL, NULL, lps); + } + else + res = decoders.Code(inStream, outStream, block, &unpSize, lps); if (res != S_OK) { if (res != S_FALSE) - return res; + { + if (res != E_NOTIMPL) + return res; + opRes = NExtract::NOperationResult::kUnsupportedMethod; + } if (opRes == NExtract::NOperationResult::kOK) opRes = NExtract::NOperationResult::kDataError; } - unpPos += block.UnpSize; + unpPos += unpSize; - if (!outStreamSpec->IsFinishedOK()) + if (!outStream->IsFinishedOK()) { - if (realMethod && opRes == NExtract::NOperationResult::kOK) + if (!block.IsZeroMethod() && opRes == NExtract::NOperationResult::kOK) opRes = NExtract::NOperationResult::kDataError; - while (outStreamSpec->GetRem() != 0) + for (unsigned k = 0;;) { - UInt64 rem = outStreamSpec->GetRem(); - UInt32 size = (UInt32)MyMin(rem, (UInt64)kZeroBufSize); + const UInt64 rem = outStream->GetRem(); + if (rem == 0) + break; + size_t size = kZeroBufSize; + if (size > rem) + size = (size_t)rem; RINOK(WriteStream(outStream, zeroBuf, size)) + k++; + if ((k & 0xfff) == 0) + { + lps->OutSize = total_UnpackSize + unpPos - outStream->GetRem(); + RINOK(lps->SetCur()) + } } } } - } - - if (needCrc && opRes == NExtract::NOperationResult::kOK) - { - if (outCrcStreamSpec->GetCRC() != item.Checksum.GetCrc32()) - opRes = NExtract::NOperationResult::kCRCError; + if (needCrc && opRes == NExtract::NOperationResult::kOK) + { + if (outCrcStream->GetCRC() != item.Checksum.GetCrc32()) + opRes = NExtract::NOperationResult::kCRCError; + } } } - outStream.Release(); + } RINOK(extractCallback->SetOperationResult(opRes)) } - return S_OK; COM_TRY_END } + + + struct CChunk { int BlockIndex; UInt64 AccessMark; - CByteBuffer Buf; + Byte *Buf; + size_t BufSize; + + void Free() + { + z7_AlignedFree(Buf); + Buf = NULL; + BufSize = 0; + } + void Alloc(size_t size) + { + Buf = (Byte *)z7_AlignedAlloc(size); + } }; -Z7_CLASS_IMP_COM_1( +Z7_CLASS_IMP_IInStream( CInStream - , IInStream ) - Z7_IFACE_COM7_IMP(ISequentialInStream) - + bool _errorMode; UInt64 _virtPos; int _latestChunk; int _latestBlock; UInt64 _accessMark; - CObjectVector _chunks; - - NCompress::NBZip2::CDecoder *bzip2CoderSpec; - CMyComPtr bzip2Coder; - - NCompress::NZlib::CDecoder *zlibCoderSpec; - CMyComPtr zlibCoder; - - CAdcDecoder *adcCoderSpec; - CMyComPtr adcCoder; - - NCompress::NLzfse::CDecoder *lzfseCoderSpec; - CMyComPtr lzfseCoder; - - CBufPtrSeqOutStream *outStreamSpec; - CMyComPtr outStream; - - CLimitedSequentialInStream *limitedStreamSpec; - CMyComPtr inStream; + UInt64 _chunks_TotalSize; + CRecordVector _chunks; public: CMyComPtr Stream; - UInt64 Size; const CFile *File; + UInt64 Size; +private: UInt64 _startPos; - HRESULT InitAndSeek(UInt64 startPos) + ~CInStream(); + CMyComPtr2 outStream; + CMyComPtr2 inStream; + CDecoders decoders; +public: + + // HRESULT + void Init(UInt64 startPos) { + _errorMode = false; _startPos = startPos; _virtPos = 0; _latestChunk = -1; _latestBlock = -1; _accessMark = 0; + _chunks_TotalSize = 0; - limitedStreamSpec = new CLimitedSequentialInStream; - inStream = limitedStreamSpec; - limitedStreamSpec->SetStream(Stream); + inStream.Create_if_Empty(); + inStream->SetStream(Stream); - outStreamSpec = new CBufPtrSeqOutStream; - outStream = outStreamSpec; - return S_OK; + outStream.Create_if_Empty(); + // return S_OK; } }; +CInStream::~CInStream() +{ + unsigned i = _chunks.Size(); + while (i) + _chunks[--i].Free(); +} + static unsigned FindBlock(const CRecordVector &blocks, UInt64 pos) { unsigned left = 0, right = blocks.Size(); for (;;) { - unsigned mid = (left + right) / 2; + const unsigned mid = (left + right) / 2; if (mid == left) return left; if (pos < blocks[mid].UnpPos) @@ -1539,7 +2073,11 @@ static unsigned FindBlock(const CRecordVector &blocks, UInt64 pos) Z7_COM7F_IMF(CInStream::Read(void *data, UInt32 size, UInt32 *processedSize)) { - COM_TRY_BEGIN + // COM_TRY_BEGIN + try { + + if (_errorMode) + return E_OUTOFMEMORY; if (processedSize) *processedSize = 0; @@ -1548,7 +2086,7 @@ Z7_COM7F_IMF(CInStream::Read(void *data, UInt32 size, UInt32 *processedSize)) if (_virtPos >= Size) return S_OK; // (Size == _virtPos) ? S_OK: E_FAIL; { - UInt64 rem = Size - _virtPos; + const UInt64 rem = Size - _virtPos; if (size > rem) size = (UInt32)rem; } @@ -1556,115 +2094,120 @@ Z7_COM7F_IMF(CInStream::Read(void *data, UInt32 size, UInt32 *processedSize)) if (_latestBlock >= 0) { const CBlock &block = File->Blocks[(unsigned)_latestBlock]; - if (_virtPos < block.UnpPos || (_virtPos - block.UnpPos) >= block.UnpSize) + if (_virtPos < block.UnpPos || + _virtPos - block.UnpPos >= File->GetUnpackSize_of_Block((unsigned)_latestBlock)) _latestBlock = -1; } if (_latestBlock < 0) { _latestChunk = -1; - unsigned blockIndex = FindBlock(File->Blocks, _virtPos); + const unsigned blockIndex = FindBlock(File->Blocks, _virtPos); const CBlock &block = File->Blocks[blockIndex]; + const UInt64 unpSize = File->GetUnpackSize_of_Block(blockIndex); - if (!block.IsZeroMethod() && block.Type != METHOD_COPY) + if (block.NeedAllocateBuffer() + && unpSize <= k_Chunk_Size_MAX) { - unsigned i; - for (i = 0; i < _chunks.Size(); i++) - if (_chunks[i].BlockIndex == (int)blockIndex) - break; - + unsigned i = 0; + { + unsigned numChunks = _chunks.Size(); + if (numChunks) + { + const CChunk *chunk = _chunks.ConstData(); + do + { + if (chunk->BlockIndex == (int)blockIndex) + break; + chunk++; + } + while (--numChunks); + i = _chunks.Size() - numChunks; + } + } if (i != _chunks.Size()) _latestChunk = (int)i; else { - const unsigned kNumChunksMax = 128; unsigned chunkIndex; - - if (_chunks.Size() != kNumChunksMax) - chunkIndex = _chunks.Add(CChunk()); - else + for (;;) { + if (_chunks.IsEmpty() || + (_chunks.Size() < k_NumChunks_MAX + && _chunks_TotalSize + unpSize <= k_Chunks_TotalSize_MAX)) + { + CChunk chunk; + chunk.Buf = NULL; + chunk.BufSize = 0; + chunk.BlockIndex = -1; + chunk.AccessMark = 0; + chunkIndex = _chunks.Add(chunk); + break; + } chunkIndex = 0; - for (i = 0; i < _chunks.Size(); i++) - if (_chunks[i].AccessMark < _chunks[chunkIndex].AccessMark) - chunkIndex = i; + if (_chunks.Size() == 1) + break; + { + const CChunk *chunks = _chunks.ConstData(); + UInt64 accessMark_min = chunks[chunkIndex].AccessMark; + const unsigned numChunks = _chunks.Size(); + for (i = 1; i < numChunks; i++) + { + if (chunks[i].AccessMark < accessMark_min) + { + chunkIndex = i; + accessMark_min = chunks[i].AccessMark; + } + } + } + { + CChunk &chunk = _chunks[chunkIndex]; + const UInt64 newTotalSize = _chunks_TotalSize - chunk.BufSize; + if (newTotalSize + unpSize <= k_Chunks_TotalSize_MAX) + break; + _chunks_TotalSize = newTotalSize; + chunk.Free(); + } + // we have called chunk.Free() before, because + // _chunks.Delete() doesn't call chunk.Free(). + _chunks.Delete(chunkIndex); + PRF(printf("\n++num_chunks=%u, _chunks_TotalSize = %u\n", (unsigned)_chunks.Size(), (unsigned)_chunks_TotalSize);) } CChunk &chunk = _chunks[chunkIndex]; chunk.BlockIndex = -1; chunk.AccessMark = 0; - if (chunk.Buf.Size() < block.UnpSize) + if (chunk.BufSize < unpSize) { - chunk.Buf.Free(); - if (block.UnpSize > ((UInt32)1 << 31)) - return E_FAIL; - chunk.Buf.Alloc((size_t)block.UnpSize); + _chunks_TotalSize -= chunk.BufSize; + chunk.Free(); + // if (unpSize > k_Chunk_Size_MAX) return E_FAIL; + chunk.Alloc((size_t)unpSize); + if (!chunk.Buf) + return E_OUTOFMEMORY; + chunk.BufSize = (size_t)unpSize; + _chunks_TotalSize += chunk.BufSize; } - outStreamSpec->Init(chunk.Buf, (size_t)block.UnpSize); - - RINOK(InStream_SeekSet(Stream, _startPos + File->StartPos + block.PackPos)) + RINOK(InStream_SeekSet(Stream, _startPos + File->StartPackPos + block.PackPos)) - limitedStreamSpec->Init(block.PackSize); - HRESULT res = S_OK; - - switch (block.Type) + inStream->Init(block.PackSize); +#ifdef Z7_DMG_USE_CACHE_FOR_COPY_BLOCKS + if (block.Type == METHOD_COPY) { - case METHOD_COPY: - if (block.PackSize != block.UnpSize) - return E_FAIL; - res = ReadStream_FAIL(inStream, chunk.Buf, (size_t)block.UnpSize); - break; - - case METHOD_ADC: - if (!adcCoder) - { - adcCoderSpec = new CAdcDecoder(); - adcCoder = adcCoderSpec; - } - res = adcCoder->Code(inStream, outStream, &block.PackSize, &block.UnpSize, NULL); - break; - - case METHOD_ZLIB: - if (!zlibCoder) - { - zlibCoderSpec = new NCompress::NZlib::CDecoder(); - zlibCoder = zlibCoderSpec; - } - res = zlibCoder->Code(inStream, outStream, NULL, NULL, NULL); - if (res == S_OK && zlibCoderSpec->GetInputProcessedSize() != block.PackSize) - res = S_FALSE; - break; - - case METHOD_BZIP2: - if (!bzip2Coder) - { - bzip2CoderSpec = new NCompress::NBZip2::CDecoder(); - bzip2Coder = bzip2CoderSpec; - } - res = bzip2Coder->Code(inStream, outStream, NULL, NULL, NULL); - if (res == S_OK && bzip2CoderSpec->GetInputProcessedSize() != block.PackSize) - res = S_FALSE; - break; - - case METHOD_LZFSE: - if (!lzfseCoder) - { - lzfseCoderSpec = new NCompress::NLzfse::CDecoder(); - lzfseCoder = lzfseCoderSpec; - } - res = lzfseCoder->Code(inStream, outStream, &block.PackSize, &block.UnpSize, NULL); - break; - - default: + if (block.PackSize != unpSize) + return E_FAIL; + RINOK(ReadStream_FAIL(inStream, chunk.Buf, (size_t)unpSize)) + } + else +#endif + { + outStream->Init(chunk.Buf, (size_t)unpSize); + RINOK(decoders.Code(inStream, outStream, block, &unpSize, NULL)) + if (outStream->GetPos() != unpSize) return E_FAIL; } - - if (res != S_OK) - return res; - if (block.Type != METHOD_COPY && outStreamSpec->GetPos() != block.UnpSize) - return E_FAIL; chunk.BlockIndex = (int)blockIndex; _latestChunk = (int)chunkIndex; } @@ -1677,29 +2220,40 @@ Z7_COM7F_IMF(CInStream::Read(void *data, UInt32 size, UInt32 *processedSize)) const CBlock &block = File->Blocks[(unsigned)_latestBlock]; const UInt64 offset = _virtPos - block.UnpPos; - const UInt64 rem = block.UnpSize - offset; - if (size > rem) - size = (UInt32)rem; - + { + const UInt64 rem = File->GetUnpackSize_of_Block((unsigned)_latestBlock) - offset; + if (size > rem) + size = (UInt32)rem; + if (size == 0) // it's unexpected case. but we check it. + return S_OK; + } HRESULT res = S_OK; - if (block.Type == METHOD_COPY) + if (block.IsZeroMethod()) + memset(data, 0, size); + else if (_latestChunk >= 0) + memcpy(data, _chunks[_latestChunk].Buf + (size_t)offset, size); + else { - RINOK(InStream_SeekSet(Stream, _startPos + File->StartPos + block.PackPos + offset)) + if (block.Type != METHOD_COPY) + return E_FAIL; + RINOK(InStream_SeekSet(Stream, _startPos + File->StartPackPos + block.PackPos + offset)) res = Stream->Read(data, size, &size); } - else if (block.IsZeroMethod()) - memset(data, 0, size); - else if (size != 0) - memcpy(data, _chunks[_latestChunk].Buf + (size_t)offset, size); _virtPos += size; if (processedSize) *processedSize = size; - return res; - COM_TRY_END + // COM_TRY_END + } + catch(...) + { + _errorMode = true; + return E_OUTOFMEMORY; + } } + Z7_COM7F_IMF(CInStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *newPosition)) { @@ -1722,29 +2276,39 @@ Z7_COM7F_IMF(CHandler::GetStream(UInt32 index, ISequentialInStream **stream)) { COM_TRY_BEGIN - #ifdef DMG_SHOW_RAW - if (index >= (UInt32)_files.Size()) +#ifdef DMG_SHOW_RAW + if (index >= _files.Size()) return S_FALSE; - #endif +#endif - CInStream *spec = new CInStream; - CMyComPtr specStream = spec; + CMyComPtr2 spec; + spec.Create_if_Empty(); spec->File = &_files[index]; const CFile &file = *spec->File; + + if (!file.IsCorrect) + return S_FALSE; FOR_VECTOR (i, file.Blocks) { const CBlock &block = file.Blocks[i]; + if (!block.NeedAllocateBuffer()) + continue; + switch (block.Type) { - case METHOD_ZERO_0: - case METHOD_ZERO_2: +#ifdef Z7_DMG_USE_CACHE_FOR_COPY_BLOCKS case METHOD_COPY: + break; +#endif case METHOD_ADC: case METHOD_ZLIB: case METHOD_BZIP2: case METHOD_LZFSE: - case METHOD_END: + case METHOD_XZ: + // case METHOD_END: + if (file.GetUnpackSize_of_Block(i) > k_Chunk_Size_MAX) + return S_FALSE; break; default: return S_FALSE; @@ -1753,8 +2317,9 @@ Z7_COM7F_IMF(CHandler::GetStream(UInt32 index, ISequentialInStream **stream)) spec->Stream = _inStream; spec->Size = spec->File->Size; - RINOK(spec->InitAndSeek(_startPos + _dataStartOffset)) - *stream = specStream.Detach(); + // RINOK( + spec->Init(_startPos + _dataForkPair.Offset); + *stream = spec.Detach(); return S_OK; COM_TRY_END diff --git a/CPP/7zip/Archive/ElfHandler.cpp b/CPP/7zip/Archive/ElfHandler.cpp index 7e1facc..e31b4ae 100644 --- a/CPP/7zip/Archive/ElfHandler.cpp +++ b/CPP/7zip/Archive/ElfHandler.cpp @@ -155,6 +155,7 @@ bool CHeader::Parse(const Byte *p) // The program header table itself. #define PT_PHDR 6 +#define PT_GNU_STACK 0x6474e551 static const CUInt32PCharPair g_SegnmentTypes[] = { @@ -167,7 +168,7 @@ static const CUInt32PCharPair g_SegnmentTypes[] = { 6, "Program header table" }, { 7, "TLS" }, { 0x6474e550, "GNU_EH_FRAME" }, - { 0x6474e551, "GNU_STACK" }, + { PT_GNU_STACK, "GNU_STACK" }, { 0x6474e552, "GNU_RELRO" } }; @@ -607,6 +608,7 @@ static const CUInt32PCharPair g_OS[] = #define k_Machine_MIPS 8 #define k_Machine_ARM 40 +#define k_Machine_RISCV 243 /* #define EF_ARM_ABIMASK 0xFF000000 @@ -640,6 +642,15 @@ static const CUInt32PCharPair g_MIPS_Flags[] = { 27, "MDMX" } }; +static const char * const g_RISCV_Flags[] = +{ + "RVC", + NULL, + NULL, + "RVE", + "TSO" +}; + // #define ET_NONE 0 #define ET_REL 1 @@ -670,6 +681,8 @@ Z7_CLASS_IMP_CHandler_IInArchive_1( CHeader _header; bool _headersError; bool _allowTail; + bool _stackFlags_Defined; + UInt32 _stackFlags; void GetSectionName(UInt32 index, NCOM::CPropVariant &prop, bool showNULL) const; HRESULT Open2(IInStream *stream); @@ -706,6 +719,7 @@ static const Byte kArcProps[] = kpidBigEndian, kpidHostOS, kpidCharacts, + kpidComment, kpidHeadersSize }; @@ -782,6 +796,23 @@ Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) s.Add_Space(); s += FlagsToString(g_MIPS_Flags, Z7_ARRAY_SIZE(g_MIPS_Flags), flags); } + else if (_header.Machine == k_Machine_RISCV) + { + s += "FLOAT_"; + const UInt32 fl = (flags >> 1) & 3; + /* + static const char * const g_RISCV_Flags_Float[] = + { "SOFT", "SINGLE", "DOUBLE", "QUAD" }; + s += g_RISCV_Flags_Float[fl]; + */ + if (fl) + s.Add_UInt32(16u << fl); + else + s += "SOFT"; + s.Add_Space(); + flags &= ~(UInt32)6; + s += FlagsToString(g_RISCV_Flags, Z7_ARRAY_SIZE(g_RISCV_Flags), flags); + } else { char sz[16]; @@ -795,6 +826,14 @@ Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) case kpidHostOS: PAIR_TO_PROP(g_OS, _header.Os, prop); break; case kpidCharacts: TYPE_TO_PROP(g_Types, _header.Type, prop); break; + case kpidComment: + if (_stackFlags_Defined) + { + AString s ("STACK: "); + s += FlagsToString(g_SegmentFlags, Z7_ARRAY_SIZE(g_SegmentFlags), _stackFlags); + prop = s; + } + break; case kpidExtension: { const char *s = NULL; @@ -913,9 +952,13 @@ HRESULT CHandler::Open2(IInStream *stream) CSegment seg; seg.Parse(p, _header.Mode64, _header.Be); seg.UpdateTotalSize(_totalSize); - if (addSegments) - if (seg.Type != PT_PHDR) - _segments.AddInReserved(seg); + if (seg.Type == PT_GNU_STACK && !_stackFlags_Defined) + { + _stackFlags = seg.Flags; + _stackFlags_Defined = true; + } + if (addSegments && seg.Type != PT_PHDR) + _segments.AddInReserved(seg); } } @@ -1002,6 +1045,7 @@ Z7_COM7F_IMF(CHandler::Close()) { _totalSize = 0; _headersError = false; + _stackFlags_Defined = false; _inStream.Release(); _segments.Clear(); @@ -1034,26 +1078,23 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, _segments[index].Size : _sections[index - _segments.Size()].GetSize(); } - extractCallback->SetTotal(totalSize); + RINOK(extractCallback->SetTotal(totalSize)) UInt64 currentTotalSize = 0; UInt64 currentItemSize; - NCompress::CCopyCoder *copyCoderSpec = new NCompress::CCopyCoder(); - CMyComPtr copyCoder = copyCoderSpec; - - CLocalProgress *lps = new CLocalProgress; - CMyComPtr progress = lps; + CMyComPtr2_Create copyCoder; + CMyComPtr2_Create lps; lps->Init(extractCallback, false); + CMyComPtr2_Create inStream; + inStream->SetStream(_inStream); - CLimitedSequentialInStream *streamSpec = new CLimitedSequentialInStream; - CMyComPtr inStream(streamSpec); - streamSpec->SetStream(_inStream); - - for (i = 0; i < numItems; i++, currentTotalSize += currentItemSize) + for (i = 0;; i++, currentTotalSize += currentItemSize) { lps->InSize = lps->OutSize = currentTotalSize; RINOK(lps->SetCur()) + if (i >= numItems) + break; const Int32 askMode = testMode ? NExtract::NAskMode::kTest : NExtract::NAskMode::kExtract; @@ -1071,18 +1112,17 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, currentItemSize = item.GetSize(); offset = item.Offset; } - - CMyComPtr outStream; - RINOK(extractCallback->GetStream(index, &outStream, askMode)) - if (!testMode && !outStream) - continue; - - RINOK(extractCallback->PrepareOperation(askMode)) - RINOK(InStream_SeekSet(_inStream, offset)) - streamSpec->Init(currentItemSize); - RINOK(copyCoder->Code(inStream, outStream, NULL, NULL, progress)) - outStream.Release(); - RINOK(extractCallback->SetOperationResult(copyCoderSpec->TotalSize == currentItemSize ? + { + CMyComPtr outStream; + RINOK(extractCallback->GetStream(index, &outStream, askMode)) + if (!testMode && !outStream) + continue; + RINOK(extractCallback->PrepareOperation(askMode)) + RINOK(InStream_SeekSet(_inStream, offset)) + inStream->Init(currentItemSize); + RINOK(copyCoder.Interface()->Code(inStream, outStream, NULL, NULL, lps)) + } + RINOK(extractCallback->SetOperationResult(copyCoder->TotalSize == currentItemSize ? NExtract::NOperationResult::kOK: NExtract::NOperationResult::kDataError)) } diff --git a/CPP/7zip/Archive/ExtHandler.cpp b/CPP/7zip/Archive/ExtHandler.cpp index f309485..5af2a92 100644 --- a/CPP/7zip/Archive/ExtHandler.cpp +++ b/CPP/7zip/Archive/ExtHandler.cpp @@ -20,6 +20,7 @@ #include "../../../C/CpuArch.h" #include "../../Common/ComTry.h" +#include "../../Common/IntToString.h" #include "../../Common/MyLinux.h" #include "../../Common/StringConvert.h" #include "../../Common/UTFConvert.h" @@ -240,15 +241,6 @@ static const char * const g_NodeFlags[] = }; -static inline char GetHex(unsigned t) { return (char)(((t < 10) ? ('0' + t) : ('A' + (t - 10)))); } - -static inline void PrintHex(unsigned v, char *s) -{ - s[0] = GetHex((v >> 4) & 0xF); - s[1] = GetHex(v & 0xF); -} - - enum { k_Type_UNKNOWN, @@ -1860,12 +1852,10 @@ Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) case kpidId: { - if (!IsEmptyData(_h.Uuid, 16)) + if (!IsEmptyData(_h.Uuid, sizeof(_h.Uuid))) { - char s[16 * 2 + 2]; - for (unsigned i = 0; i < 16; i++) - PrintHex(_h.Uuid[i], s + i * 2); - s[16 * 2] = 0; + char s[sizeof(_h.Uuid) * 2 + 2]; + ConvertDataToHex_Lower(s, _h.Uuid, sizeof(_h.Uuid)); prop = s; } break; @@ -2602,7 +2592,7 @@ HRESULT CHandler::GetStream_Node(unsigned nodeIndex, ISequentialInStream **strea CMyComPtr streamTemp; - UInt64 numBlocks64 = (node.FileSize + (UInt64)(((UInt32)1 << _h.BlockBits) - 1)) >> _h.BlockBits; + const UInt64 numBlocks64 = (node.FileSize + (UInt64)(((UInt32)1 << _h.BlockBits) - 1)) >> _h.BlockBits; if (node.IsFlags_EXTENTS()) { @@ -2729,27 +2719,25 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, totalSize += node.FileSize; } - extractCallback->SetTotal(totalSize); + RINOK(extractCallback->SetTotal(totalSize)) UInt64 totalPackSize; totalSize = totalPackSize = 0; - NCompress::CCopyCoder *copyCoderSpec = new NCompress::CCopyCoder(); - CMyComPtr copyCoder = copyCoderSpec; - - CLocalProgress *lps = new CLocalProgress; - CMyComPtr progress = lps; + CMyComPtr2_Create lps; lps->Init(extractCallback, false); + CMyComPtr2_Create copyCoder; for (i = 0;; i++) { lps->InSize = totalPackSize; lps->OutSize = totalSize; RINOK(lps->SetCur()) - - if (i == numItems) + if (i >= numItems) break; + int opRes; + { CMyComPtr outStream; const Int32 askMode = testMode ? NExtract::NAskMode::kTest : @@ -2786,7 +2774,7 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, continue; RINOK(extractCallback->PrepareOperation(askMode)) - int res = NExtract::NOperationResult::kDataError; + opRes = NExtract::NOperationResult::kDataError; { CMyComPtr inSeqStream; HRESULT hres = GetStream(index, &inSeqStream); @@ -2794,21 +2782,21 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, { if (hres == E_OUTOFMEMORY) return hres; - res = NExtract::NOperationResult::kUnsupportedMethod; + opRes = NExtract::NOperationResult::kUnsupportedMethod; } else { RINOK(hres) { - hres = copyCoder->Code(inSeqStream, outStream, NULL, NULL, progress); + hres = copyCoder.Interface()->Code(inSeqStream, outStream, NULL, NULL, lps); if (hres == S_OK) { - if (copyCoderSpec->TotalSize == unpackSize) - res = NExtract::NOperationResult::kOK; + if (copyCoder->TotalSize == unpackSize) + opRes = NExtract::NOperationResult::kOK; } else if (hres == E_NOTIMPL) { - res = NExtract::NOperationResult::kUnsupportedMethod; + opRes = NExtract::NOperationResult::kUnsupportedMethod; } else if (hres != S_FALSE) { @@ -2817,7 +2805,8 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, } } } - RINOK(extractCallback->SetOperationResult(res)) + } + RINOK(extractCallback->SetOperationResult(opRes)) } return S_OK; diff --git a/CPP/7zip/Archive/FatHandler.cpp b/CPP/7zip/Archive/FatHandler.cpp index 4a843a8..b31ee4f 100644 --- a/CPP/7zip/Archive/FatHandler.cpp +++ b/CPP/7zip/Archive/FatHandler.cpp @@ -26,6 +26,8 @@ #define Get16(p) GetUi16(p) #define Get32(p) GetUi32(p) +#define Get16a(p) GetUi16a(p) +#define Get32a(p) GetUi32a(p) #define PRF(x) /* x */ @@ -57,6 +59,7 @@ struct CHeader UInt32 NumHiddenSectors; bool VolFieldsDefined; + bool HeadersWarning; UInt32 VolId; // Byte VolName[11]; @@ -125,6 +128,8 @@ bool CHeader::Parse(const Byte *p) if (p[0x1FE] != 0x55 || p[0x1FF] != 0xAA) return false; + HeadersWarning = false; + int codeOffset = 0; switch (p[0]) { @@ -259,8 +264,22 @@ bool CHeader::Parse(const Byte *p) } FatSize = numClusters + 2; - if (FatSize > BadCluster || CalcFatSizeInSectors() > NumFatSectors) + if (FatSize > BadCluster) return false; + if (CalcFatSizeInSectors() > NumFatSectors) + { + /* some third-party program can create such FAT image, where + size of FAT table (NumFatSectors from headers) is smaller than + required value that is calculated from calculated (FatSize) value. + Another FAT unpackers probably ignore that error. + v23.02: we also ignore that error, and + we recalculate (FatSize) value from (NumFatSectors). + New (FatSize) will be smaller than original "full" (FatSize) value. + So we will have some unused clusters at the end of archive. + */ + FatSize = (UInt32)(((UInt64)NumFatSectors << (3 + SectorSizeLog)) / NumFatBits); + HeadersWarning = true; + } return true; } @@ -410,7 +429,7 @@ HRESULT CDatabase::OpenProgressFat(bool changeTotal) return S_OK; if (changeTotal) { - UInt64 numTotalBytes = (Header.CalcFatSizeInSectors() << Header.SectorSizeLog) + + const UInt64 numTotalBytes = (Header.CalcFatSizeInSectors() << Header.SectorSizeLog) + ((UInt64)(Header.FatSize - NumFreeClusters) << Header.ClusterSizeLog); RINOK(OpenCallback->SetTotal(NULL, &numTotalBytes)) } @@ -494,7 +513,7 @@ HRESULT CDatabase::ReadDir(Int32 parent, UInt32 cluster, unsigned level) return S_FALSE; PRF(printf("\nCluster = %4X", cluster)); RINOK(SeekToCluster(cluster)) - UInt32 newCluster = Fat[cluster]; + const UInt32 newCluster = Fat[cluster]; if ((newCluster & kFatItemUsedByDirMask) != 0) return S_FALSE; Fat[cluster] |= kFatItemUsedByDirMask; @@ -650,15 +669,18 @@ HRESULT CDatabase::Open() if (Header.IsFat32()) { - SeekToSector(Header.FsInfoSector); - RINOK(ReadStream_FALSE(InStream, buf, kHeaderSize)) - if (buf[0x1FE] != 0x55 || buf[0x1FF] != 0xAA) - return S_FALSE; - if (Get32(buf) == 0x41615252 && Get32(buf + 484) == 0x61417272) + if (((UInt32)Header.FsInfoSector << Header.SectorSizeLog) + kHeaderSize <= fileSize + && SeekToSector(Header.FsInfoSector) == S_OK + && ReadStream_FALSE(InStream, buf, kHeaderSize) == S_OK + && 0xaa550000 == Get32(buf + 508) + && 0x41615252 == Get32(buf) + && 0x61417272 == Get32(buf + 484)) { NumFreeClusters = Get32(buf + 488); numFreeClustersDefined = (NumFreeClusters <= Header.FatSize); } + else + Header.HeadersWarning = true; } } @@ -675,30 +697,35 @@ HRESULT CDatabase::Open() { const UInt32 kBufSize = (1 << 15); byteBuf.Alloc(kBufSize); - for (UInt32 i = 0; i < Header.FatSize;) + for (UInt32 i = 0;;) { UInt32 size = Header.FatSize - i; + if (size == 0) + break; const UInt32 kBufSize32 = kBufSize / 4; if (size > kBufSize32) size = kBufSize32; - UInt32 readSize = Header.SizeToSectors(size * 4) << Header.SectorSizeLog; + const UInt32 readSize = Header.SizeToSectors(size * 4) << Header.SectorSizeLog; RINOK(ReadStream_FALSE(InStream, byteBuf, readSize)) NumCurUsedBytes += readSize; const UInt32 *src = (const UInt32 *)(const void *)(const Byte *)byteBuf; UInt32 *dest = Fat + i; + const UInt32 *srcLim = src + size; if (numFreeClustersDefined) - for (UInt32 j = 0; j < size; j++) - dest[j] = Get32(src + j) & 0x0FFFFFFF; + do + *dest++ = Get32a(src) & 0x0FFFFFFF; + while (++src != srcLim); else { UInt32 numFreeClusters = 0; - for (UInt32 j = 0; j < size; j++) + do { - UInt32 v = Get32(src + j) & 0x0FFFFFFF; + const UInt32 v = Get32a(src) & 0x0FFFFFFF; + *dest++ = v; numFreeClusters += (UInt32)(v - 1) >> 31; - dest[j] = v; } + while (++src != srcLim); NumFreeClusters += numFreeClusters; } i += size; @@ -715,11 +742,11 @@ HRESULT CDatabase::Open() byteBuf.Alloc(kBufSize); Byte *p = byteBuf; RINOK(ReadStream_FALSE(InStream, p, kBufSize)) - UInt32 fatSize = Header.FatSize; + const UInt32 fatSize = Header.FatSize; UInt32 *fat = &Fat[0]; if (Header.NumFatBits == 16) for (UInt32 j = 0; j < fatSize; j++) - fat[j] = Get16(p + j * 2); + fat[j] = Get16a(p + j * 2); else for (UInt32 j = 0; j < fatSize; j++) fat[j] = (Get16(p + j * 3 / 2) >> ((j & 1) << 2)) & 0xFFF; @@ -773,7 +800,7 @@ Z7_COM7F_IMF(CHandler::GetStream(UInt32 index, ISequentialInStream **stream)) streamSpec->BlockSizeLog = Header.ClusterSizeLog; streamSpec->Size = item.Size; - UInt32 numClusters = Header.GetNumClusters(item.Size); + const UInt32 numClusters = Header.GetNumClusters(item.Size); streamSpec->Vector.ClearAndReserve(numClusters); UInt32 cluster = item.Cluster; UInt32 size = item.Size; @@ -785,7 +812,7 @@ Z7_COM7F_IMF(CHandler::GetStream(UInt32 index, ISequentialInStream **stream)) } else { - UInt32 clusterSize = Header.ClusterSize(); + const UInt32 clusterSize = Header.ClusterSize(); for (;; size -= clusterSize) { if (!Header.IsValidCluster(cluster)) @@ -907,6 +934,14 @@ Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) // case kpidVolName: if (Header.VolFieldsDefined) STRING_TO_PROP(Header.VolName, prop); break; // case kpidFileSysType: if (Header.VolFieldsDefined) STRING_TO_PROP(Header.FileSys, prop); break; // case kpidHiddenSectors: prop = Header.NumHiddenSectors; break; + case kpidWarningFlags: + { + UInt32 v = 0; + if (Header.HeadersWarning) v |= kpv_ErrorFlags_HeadersError; + if (v != 0) + prop = v; + break; + } } prop.Detach(value); return S_OK; diff --git a/CPP/7zip/Archive/GptHandler.cpp b/CPP/7zip/Archive/GptHandler.cpp index 78c76cf..23a1db6 100644 --- a/CPP/7zip/Archive/GptHandler.cpp +++ b/CPP/7zip/Archive/GptHandler.cpp @@ -24,6 +24,10 @@ using namespace NWindows; namespace NArchive { +namespace NMbr { +const char *GetFileSystem(ISequentialInStream *stream, UInt64 partitionSize); +} + namespace NFat { API_FUNC_IsArc IsArc_Fat(const Byte *p, size_t size); } @@ -34,8 +38,6 @@ static const unsigned k_SignatureSize = 12; static const Byte k_Signature[k_SignatureSize] = { 'E', 'F', 'I', ' ', 'P', 'A', 'R', 'T', 0, 0, 1, 0 }; -static const UInt32 kSectorSize = 512; - static const CUInt32PCharPair g_PartitionFlags[] = { { 0, "Sys" }, @@ -66,9 +68,9 @@ struct CPartition return true; } - UInt64 GetSize() const { return (LastLba - FirstLba + 1) * kSectorSize; } - UInt64 GetPos() const { return FirstLba * kSectorSize; } - UInt64 GetEnd() const { return (LastLba + 1) * kSectorSize; } + UInt64 GetSize(unsigned sectorSizeLog) const { return (LastLba - FirstLba + 1) << sectorSizeLog; } + UInt64 GetPos(unsigned sectorSizeLog) const { return FirstLba << sectorSizeLog; } + UInt64 GetEnd(unsigned sectorSizeLog) const { return (LastLba + 1) << sectorSizeLog; } void Parse(const Byte *p) { @@ -144,6 +146,7 @@ Z7_class_CHandler_final: public CHandlerCont CRecordVector _items; UInt64 _totalSize; + unsigned _sectorSizeLog; Byte Guid[16]; CByteBuffer _buffer; @@ -153,8 +156,8 @@ Z7_class_CHandler_final: public CHandlerCont virtual int GetItem_ExtractInfo(UInt32 index, UInt64 &pos, UInt64 &size) const Z7_override { const CPartition &item = _items[index]; - pos = item.GetPos(); - size = item.GetSize(); + pos = item.GetPos(_sectorSizeLog); + size = item.GetSize(_sectorSizeLog); return NExtract::NOperationResult::kOK; } }; @@ -162,22 +165,32 @@ Z7_class_CHandler_final: public CHandlerCont HRESULT CHandler::Open2(IInStream *stream) { - _buffer.Alloc(kSectorSize * 2); - RINOK(ReadStream_FALSE(stream, _buffer, kSectorSize * 2)) - + const unsigned kBufSize = 2 << 12; + _buffer.Alloc(kBufSize); + RINOK(ReadStream_FALSE(stream, _buffer, kBufSize)) const Byte *buf = _buffer; if (buf[0x1FE] != 0x55 || buf[0x1FF] != 0xAA) return S_FALSE; - - buf += kSectorSize; - if (memcmp(buf, k_Signature, k_SignatureSize) != 0) - return S_FALSE; + { + for (unsigned sectorSizeLog = 9;; sectorSizeLog += 3) + { + if (sectorSizeLog > 12) + return S_FALSE; + if (memcmp(buf + ((size_t)1 << sectorSizeLog), k_Signature, k_SignatureSize) == 0) + { + buf += ((size_t)1 << sectorSizeLog); + _sectorSizeLog = sectorSizeLog; + break; + } + } + } + const UInt32 kSectorSize = 1u << _sectorSizeLog; { // if (Get32(buf + 8) != 0x10000) return S_FALSE; // revision - UInt32 headerSize = Get32(buf + 12); // = 0x5C usually + const UInt32 headerSize = Get32(buf + 12); // = 0x5C usually if (headerSize > kSectorSize) return S_FALSE; - UInt32 crc = Get32(buf + 0x10); + const UInt32 crc = Get32(buf + 0x10); SetUi32(_buffer + kSectorSize + 0x10, 0) if (CrcCalc(_buffer + kSectorSize, headerSize) != crc) return S_FALSE; @@ -191,18 +204,15 @@ HRESULT CHandler::Open2(IInStream *stream) // UInt64 lastUsableLba = Get64(buf + 0x30); memcpy(Guid, buf + 0x38, 16); const UInt64 tableLba = Get64(buf + 0x48); - if (tableLba < 2) + if (tableLba < 2 || (tableLba >> (63 - _sectorSizeLog)) != 0) return S_FALSE; const UInt32 numEntries = Get32(buf + 0x50); + if (numEntries > (1 << 16)) + return S_FALSE; const UInt32 entrySize = Get32(buf + 0x54); // = 128 usually - const UInt32 entriesCrc = Get32(buf + 0x58); - - if (entrySize < 128 - || entrySize > (1 << 12) - || numEntries > (1 << 16) - || tableLba < 2 - || tableLba >= ((UInt64)1 << (64 - 10))) + if (entrySize < 128 || entrySize > (1 << 12)) return S_FALSE; + const UInt32 entriesCrc = Get32(buf + 0x58); const UInt32 tableSize = entrySize * numEntries; const UInt32 tableSizeAligned = (tableSize + kSectorSize - 1) & ~(kSectorSize - 1); @@ -222,18 +232,24 @@ HRESULT CHandler::Open2(IInStream *stream) item.Parse(_buffer + i * entrySize); if (item.IsUnused()) continue; - UInt64 endPos = item.GetEnd(); + if (item.LastLba < item.FirstLba) + return S_FALSE; + if ((item.LastLba >> (63 - _sectorSizeLog)) != 0) + return S_FALSE; + const UInt64 endPos = item.GetEnd(_sectorSizeLog); if (_totalSize < endPos) _totalSize = endPos; _items.Add(item); } - + + _buffer.Free(); { + if ((backupLba >> (63 - _sectorSizeLog)) != 0) + return S_FALSE; const UInt64 end = (backupLba + 1) * kSectorSize; if (_totalSize < end) _totalSize = end; } - { UInt64 fileEnd; RINOK(InStream_GetSize_SeekToEnd(stream, fileEnd)) @@ -258,27 +274,6 @@ HRESULT CHandler::Open2(IInStream *stream) } - -static const unsigned k_Ntfs_Fat_HeaderSize = 512; - -static const Byte k_NtfsSignature[] = { 'N', 'T', 'F', 'S', ' ', ' ', ' ', ' ', 0 }; - -static bool IsNtfs(const Byte *p) -{ - if (p[0x1FE] != 0x55 || p[0x1FF] != 0xAA) - return false; - if (memcmp(p + 3, k_NtfsSignature, Z7_ARRAY_SIZE(k_NtfsSignature)) != 0) - return false; - switch (p[0]) - { - case 0xE9: /* codeOffset = 3 + (Int16)Get16(p + 1); */ break; - case 0xEB: if (p[2] != 0x90) return false; /* codeOffset = 2 + (int)(signed char)p[1]; */ break; - default: return false; - } - return true; -} - - Z7_COM7F_IMF(CHandler::Open(IInStream *stream, const UInt64 * /* maxCheckStartPosition */, IArchiveOpenCallback * /* openArchiveCallback */)) @@ -307,20 +302,9 @@ Z7_COM7F_IMF(CHandler::Open(IInStream *stream, // ((IInArchiveGetStream *)this)-> GetStream(fileIndex, &inStream) == S_OK && inStream) { - Byte temp[k_Ntfs_Fat_HeaderSize]; - if (ReadStream_FAIL(inStream, temp, k_Ntfs_Fat_HeaderSize) == S_OK) - { - if (IsNtfs(temp)) - { - item.Ext = "ntfs"; - continue; - } - if (NFat::IsArc_Fat(temp, k_Ntfs_Fat_HeaderSize) == k_IsArc_Res_YES) - { - item.Ext = "fat"; - continue; - } - } + const char *fs = NMbr::GetFileSystem(inStream, item.GetSize(_sectorSizeLog)); + if (fs) + item.Ext = fs; } } } @@ -331,6 +315,7 @@ Z7_COM7F_IMF(CHandler::Open(IInStream *stream, Z7_COM7F_IMF(CHandler::Close()) { + _sectorSizeLog = 0; _totalSize = 0; memset(Guid, 0, sizeof(Guid)); _items.Clear(); @@ -350,6 +335,7 @@ static const Byte kProps[] = static const Byte kArcProps[] = { + kpidSectorSize, kpidId }; @@ -369,6 +355,7 @@ Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) break; } case kpidPhySize: prop = _totalSize; break; + case kpidSectorSize: prop = (UInt32)((UInt32)1 << _sectorSizeLog); break; case kpidId: { char s[48]; @@ -420,15 +407,22 @@ Z7_COM7F_IMF(CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val } { s.Add_Dot(); - s += (item.Ext ? item.Ext : "img"); + if (item.Ext) + { + AString fs (item.Ext); + fs.MakeLower_Ascii(); + s += fs; + } + else + s += "img"; } prop = s; break; } case kpidSize: - case kpidPackSize: prop = item.GetSize(); break; - case kpidOffset: prop = item.GetPos(); break; + case kpidPackSize: prop = item.GetSize(_sectorSizeLog); break; + case kpidOffset: prop = item.GetPos(_sectorSizeLog); break; case kpidFileSystem: { @@ -462,10 +456,11 @@ Z7_COM7F_IMF(CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val COM_TRY_END } +// we suppport signature only for 512-bytes sector. REGISTER_ARC_I( "GPT", "gpt mbr", NULL, 0xCB, k_Signature, - kSectorSize, + 1 << 9, 0, NULL) diff --git a/CPP/7zip/Archive/GzHandler.cpp b/CPP/7zip/Archive/GzHandler.cpp index ad9a074..ca9d246 100644 --- a/CPP/7zip/Archive/GzHandler.cpp +++ b/CPP/7zip/Archive/GzHandler.cpp @@ -10,7 +10,6 @@ #include "../../Common/Defs.h" #include "../../Common/StringConvert.h" -#include "../../Windows/PropVariant.h" #include "../../Windows/PropVariantUtils.h" #include "../../Windows/TimeUtils.h" @@ -206,13 +205,13 @@ static HRESULT ReadString(NDecoder::CCOMCoder *stream, AString &s, size_t limit s.Empty(); for (size_t i = 0; i < limit; i++) { - Byte b = stream->ReadAlignedByte(); + const Byte b = stream->ReadAlignedByte(); if (stream->InputEofError()) return S_FALSE; // crc = CRC_UPDATE_BYTE(crc, b); if (b == 0) return S_OK; - s += (char)b; + s.Add_Char((char)b); } return S_FALSE; } @@ -229,7 +228,7 @@ static UInt32 Is_Deflate(const Byte *p, size_t size) return k_IsArc_Res_NO; if (type == 0) { - // Stored (uncompreessed data) + // Stored (uncompreessed data) if ((b >> 3) != 0) return k_IsArc_Res_NO; if (size < 4) @@ -261,11 +260,11 @@ API_FUNC_static_IsArc IsArc_Gz(const Byte *p, size_t size) p[2] != kSignature_2) return k_IsArc_Res_NO; - Byte flags = p[3]; + const Byte flags = p[3]; if ((flags & NFlags::kReserved) != 0) return k_IsArc_Res_NO; - Byte extraFlags = p[8]; + const Byte extraFlags = p[8]; // maybe that flag can have another values for some gz archives? if (extraFlags != 0 && extraFlags != NExtraFlags::kMaximum && @@ -288,7 +287,7 @@ API_FUNC_static_IsArc IsArc_Gz(const Byte *p, size_t size) return k_IsArc_Res_NO; if (size < 4) return k_IsArc_Res_NEED_MORE; - unsigned len = GetUi16(p + 2); + const unsigned len = GetUi16(p + 2); size -= 4; xlen -= 4; p += 4; @@ -469,24 +468,19 @@ Z7_CLASS_IMP_CHandler_IInArchive_3( UInt64 _headerSize; // only start header (without footer) CMyComPtr _stream; - CMyComPtr _decoder; - NDecoder::CCOMCoder *_decoderSpec; + CMyComPtr2 _decoder; CSingleMethodProps _props; CHandlerTimeOptions _timeOptions; public: CHandler(): - _isArc(false), - _decoderSpec(NULL) + _isArc(false) {} void CreateDecoder() { - if (_decoder) - return; - _decoderSpec = new NDecoder::CCOMCoder; - _decoder = _decoderSpec; + _decoder.Create_if_Empty(); } }; @@ -538,6 +532,7 @@ Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) prop = s; } break; + default: break; } prop.Detach(value); return S_OK; @@ -587,6 +582,7 @@ Z7_COM7F_IMF(CHandler::GetProperty(UInt32 /* index */, PROPID propID, PROPVARIAN } case kpidHostOS: TYPE_TO_PROP(kHostOSes, _item.HostOS, prop); break; case kpidCRC: if (_stream) prop = _item.Crc; break; + default: break; } prop.Detach(value); return S_OK; @@ -640,12 +636,12 @@ Z7_COM7F_IMF(CHandler::OpenSeq(ISequentialInStream *stream)) { Close(); CreateDecoder(); - _decoderSpec->SetInStream(stream); - _decoderSpec->InitInStream(true); - RINOK(_item.ReadHeader(_decoderSpec)) - if (_decoderSpec->InputEofError()) + _decoder->SetInStream(stream); + _decoder->InitInStream(true); + RINOK(_item.ReadHeader(_decoder.ClsPtr())) + if (_decoder->InputEofError()) return S_FALSE; - _headerSize = _decoderSpec->GetInputProcessedSize(); + _headerSize = _decoder->GetInputProcessedSize(); _isArc = true; return S_OK; } @@ -669,7 +665,7 @@ Z7_COM7F_IMF(CHandler::Close()) _stream.Release(); if (_decoder) - _decoderSpec->ReleaseInStream(); + _decoder->ReleaseInStream(); return S_OK; } @@ -683,9 +679,11 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, return E_INVALIDARG; if (_packSize_Defined) - extractCallback->SetTotal(_packSize); + RINOK(extractCallback->SetTotal(_packSize)) // UInt64 currentTotalPacked = 0; // RINOK(extractCallback->SetCompleted(¤tTotalPacked)); + Int32 retResult; + { CMyComPtr realOutStream; const Int32 askMode = testMode ? NExtract::NAskMode::kTest : @@ -694,18 +692,16 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, if (!testMode && !realOutStream) return S_OK; - extractCallback->PrepareOperation(askMode); + RINOK(extractCallback->PrepareOperation(askMode)) CreateDecoder(); - COutStreamWithCRC *outStreamSpec = new COutStreamWithCRC; - CMyComPtr outStream(outStreamSpec); - outStreamSpec->SetStream(realOutStream); - outStreamSpec->Init(); - realOutStream.Release(); + CMyComPtr2_Create outStream; + outStream->SetStream(realOutStream); + outStream->Init(); + // realOutStream.Release(); - CLocalProgress *lps = new CLocalProgress; - CMyComPtr progress = lps; + CMyComPtr2_Create lps; lps->Init(extractCallback, true); bool needReadFirstItem = _needSeekToStart; @@ -715,7 +711,7 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, if (!_stream) return E_FAIL; RINOK(InStream_SeekToBegin(_stream)) - _decoderSpec->InitInStream(true); + _decoder->InitInStream(true); // printf("\nSeek"); } else @@ -723,7 +719,7 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, bool firstItem = true; - UInt64 packSize = _decoderSpec->GetInputProcessedSize(); + UInt64 packSize = _decoder->GetInputProcessedSize(); // printf("\npackSize = %d", (unsigned)packSize); UInt64 unpackedSize = 0; @@ -746,12 +742,12 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, if (!firstItem || needReadFirstItem) { - result = item.ReadHeader(_decoderSpec); + result = item.ReadHeader(_decoder.ClsPtr()); if (result != S_OK && result != S_FALSE) return result; - if (_decoderSpec->InputEofError()) + if (_decoder->InputEofError()) result = S_FALSE; if (result != S_OK && firstItem) @@ -760,7 +756,7 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, break; } - if (packSize == _decoderSpec->GetStreamSize()) + if (packSize == _decoder->GetStreamSize()) { result = S_OK; break; @@ -776,20 +772,20 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, numStreams++; firstItem = false; - UInt64 startOffset = outStreamSpec->GetSize(); - outStreamSpec->InitCRC(); + const UInt64 startOffset = outStream->GetSize(); + outStream->InitCRC(); - result = _decoderSpec->CodeResume(outStream, NULL, progress); + result = _decoder->CodeResume(outStream, NULL, lps); - packSize = _decoderSpec->GetInputProcessedSize(); - unpackedSize = outStreamSpec->GetSize(); + packSize = _decoder->GetInputProcessedSize(); + unpackedSize = outStream->GetSize(); if (result != S_OK && result != S_FALSE) return result; - if (_decoderSpec->InputEofError()) + if (_decoder->InputEofError()) { - packSize = _decoderSpec->GetStreamSize(); + packSize = _decoder->GetStreamSize(); _needMoreInput = true; result = S_FALSE; } @@ -797,18 +793,18 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, if (result != S_OK) break; - _decoderSpec->AlignToByte(); + _decoder->AlignToByte(); - result = item.ReadFooter1(_decoderSpec); + result = item.ReadFooter1(_decoder.ClsPtr()); - packSize = _decoderSpec->GetInputProcessedSize(); + packSize = _decoder->GetInputProcessedSize(); if (result != S_OK && result != S_FALSE) return result; if (result != S_OK) { - if (_decoderSpec->InputEofError()) + if (_decoder->InputEofError()) { _needMoreInput = true; result = S_FALSE; @@ -816,7 +812,7 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, break; } - if (item.Crc != outStreamSpec->GetCRC() || + if (item.Crc != outStream->GetCRC() || item.Size32 != (UInt32)(unpackedSize - startOffset)) { crcError = true; @@ -840,9 +836,9 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, _numStreams_Defined = true; } - outStream.Release(); + // outStream.Release(); - Int32 retResult = NExtract::NOperationResult::kDataError; + retResult = NExtract::NOperationResult::kDataError; if (!_isArc) retResult = NExtract::NOperationResult::kIsNotArc; @@ -858,10 +854,9 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, retResult = NExtract::NOperationResult::kOK; else return result; + } return extractCallback->SetOperationResult(retResult); - - COM_TRY_END } @@ -966,13 +961,11 @@ static HRESULT UpdateArchive( RINOK(updateCallback->SetTotal(unpackSize)) RINOK(updateCallback->SetCompleted(&complexity)) - CSequentialInStreamWithCRC *inStreamSpec = new CSequentialInStreamWithCRC; - CMyComPtr crcStream(inStreamSpec); - inStreamSpec->SetStream(fileInStream); - inStreamSpec->Init(); + CMyComPtr2_Create crcStream; + crcStream->SetStream(fileInStream); + crcStream->Init(); - CLocalProgress *lps = new CLocalProgress; - CMyComPtr progress = lps; + CMyComPtr2_Create lps; lps->Init(updateCallback, true); item.ExtraFlags = props.GetLevel() >= 7 ? @@ -983,13 +976,13 @@ static HRESULT UpdateArchive( RINOK(item.WriteHeader(outStream)) - NEncoder::CCOMCoder *deflateEncoderSpec = new NEncoder::CCOMCoder; - CMyComPtr deflateEncoder = deflateEncoderSpec; - RINOK(props.SetCoderProps(deflateEncoderSpec, NULL)) - RINOK(deflateEncoder->Code(crcStream, outStream, NULL, NULL, progress)) + CMyComPtr2_Create deflateEncoder; + + RINOK(props.SetCoderProps(deflateEncoder.ClsPtr(), NULL)) + RINOK(deflateEncoder.Interface()->Code(crcStream, outStream, NULL, NULL, lps)) - item.Crc = inStreamSpec->GetCRC(); - unpackSizeReal = inStreamSpec->GetSize(); + item.Crc = crcStream->GetCRC(); + unpackSizeReal = crcStream->GetSize(); item.Size32 = (UInt32)unpackSizeReal; RINOK(item.WriteFooter(outStream)) } @@ -1124,8 +1117,7 @@ Z7_COM7F_IMF(CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt if (!_stream) return E_NOTIMPL; - CLocalProgress *lps = new CLocalProgress; - CMyComPtr progress = lps; + CMyComPtr2_Create lps; lps->Init(updateCallback, true); Z7_DECL_CMyComPtr_QI_FROM( @@ -1156,7 +1148,7 @@ Z7_COM7F_IMF(CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt NULL); // unpacksize */ - return NCompress::CopyStream(_stream, outStream, progress); + return NCompress::CopyStream(_stream, outStream, lps); COM_TRY_END } diff --git a/CPP/7zip/Archive/HandlerCont.cpp b/CPP/7zip/Archive/HandlerCont.cpp index b4524a4..6a0eee2 100644 --- a/CPP/7zip/Archive/HandlerCont.cpp +++ b/CPP/7zip/Archive/HandlerCont.cpp @@ -37,26 +37,24 @@ Z7_COM7F_IMF(CHandlerCont::Extract(const UInt32 *indices, UInt32 numItems, GetItem_ExtractInfo(allFilesMode ? i : indices[i], pos, size); totalSize += size; } - extractCallback->SetTotal(totalSize); + RINOK(extractCallback->SetTotal(totalSize)) totalSize = 0; - NCompress::CCopyCoder *copyCoderSpec = new NCompress::CCopyCoder(); - CMyComPtr copyCoder = copyCoderSpec; - - CLocalProgress *lps = new CLocalProgress; - CMyComPtr progress = lps; + CMyComPtr2_Create lps; lps->Init(extractCallback, false); - - CLimitedSequentialInStream *streamSpec = new CLimitedSequentialInStream; - CMyComPtr inStream(streamSpec); + CMyComPtr2_Create streamSpec; streamSpec->SetStream(_stream); + CMyComPtr2_Create copyCoder; - for (i = 0; i < numItems; i++) + for (i = 0;; i++) { lps->InSize = totalSize; lps->OutSize = totalSize; RINOK(lps->SetCur()) + if (i >= numItems) + break; + CMyComPtr outStream; const Int32 askMode = testMode ? NExtract::NAskMode::kTest : @@ -78,13 +76,12 @@ Z7_COM7F_IMF(CHandlerCont::Extract(const UInt32 *indices, UInt32 numItems, RINOK(InStream_SeekSet(_stream, pos)) streamSpec->Init(size); - RINOK(copyCoder->Code(inStream, outStream, NULL, NULL, progress)) + RINOK(copyCoder.Interface()->Code(streamSpec, outStream, NULL, NULL, lps)) opRes = NExtract::NOperationResult::kDataError; - - if (copyCoderSpec->TotalSize == size) + if (copyCoder->TotalSize == size) opRes = NExtract::NOperationResult::kOK; - else if (copyCoderSpec->TotalSize < size) + else if (copyCoder->TotalSize < size) opRes = NExtract::NOperationResult::kUnexpectedEnd; } @@ -135,23 +132,28 @@ Z7_COM7F_IMF(CHandlerImg::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *newPosit return S_OK; } -static const Byte k_GDP_Signature[] = { 'E', 'F', 'I', ' ', 'P', 'A', 'R', 'T' }; +static const Byte k_GDP_Signature[] = + { 'E', 'F', 'I', ' ', 'P', 'A', 'R', 'T', 0, 0, 1, 0 }; // static const Byte k_Ext_Signature[] = { 0x53, 0xEF }; // static const unsigned k_Ext_Signature_offset = 0x438; static const char *GetImgExt(ISequentialInStream *stream) { - const size_t kHeaderSize = 1 << 11; + // const size_t kHeaderSize_for_Ext = (1 << 11); // for ext + const size_t kHeaderSize = 2 << 12; // for 4 KB sector GPT Byte buf[kHeaderSize]; - if (ReadStream_FAIL(stream, buf, kHeaderSize) == S_OK) + size_t processed = kHeaderSize; + if (ReadStream(stream, buf, &processed) == S_OK) { + if (processed >= kHeaderSize) if (buf[0x1FE] == 0x55 && buf[0x1FF] == 0xAA) { - if (memcmp(buf + 512, k_GDP_Signature, sizeof(k_GDP_Signature)) == 0) - return "gpt"; + for (unsigned k = (1 << 9); k <= (1u << 12); k <<= 3) + if (memcmp(buf + k, k_GDP_Signature, sizeof(k_GDP_Signature)) == 0) + return "gpt"; return "mbr"; } - if (NExt::IsArc_Ext(buf, kHeaderSize) == k_IsArc_Res_YES) + if (NExt::IsArc_Ext(buf, processed) == k_IsArc_Res_YES) return "ext"; } return NULL; @@ -280,13 +282,12 @@ Z7_COM7F_IMF(CHandlerImg::Extract(const UInt32 *indices, UInt32 numItems, progress = imgProgress; } - NCompress::CCopyCoder *copyCoderSpec = new NCompress::CCopyCoder(); - CMyComPtr copyCoder = copyCoderSpec; + CMyComPtr2_Create copyCoder; - hres = copyCoder->Code(inStream, outStream, NULL, &_size, progress); + hres = copyCoder.Interface()->Code(inStream, outStream, NULL, &_size, progress); if (hres == S_OK) { - if (copyCoderSpec->TotalSize == _size) + if (copyCoder->TotalSize == _size) opRes = NExtract::NOperationResult::kOK; if (_stream_unavailData) @@ -295,7 +296,7 @@ Z7_COM7F_IMF(CHandlerImg::Extract(const UInt32 *indices, UInt32 numItems, opRes = NExtract::NOperationResult::kUnsupportedMethod; else if (_stream_dataError) opRes = NExtract::NOperationResult::kDataError; - else if (copyCoderSpec->TotalSize < _size) + else if (copyCoder->TotalSize < _size) opRes = NExtract::NOperationResult::kUnexpectedEnd; } } @@ -317,30 +318,4 @@ Z7_COM7F_IMF(CHandlerImg::Extract(const UInt32 *indices, UInt32 numItems, COM_TRY_END } - -HRESULT ReadZeroTail(ISequentialInStream *stream, bool &areThereNonZeros, UInt64 &numZeros, UInt64 maxSize) -{ - areThereNonZeros = false; - numZeros = 0; - const size_t kBufSize = 1 << 11; - Byte buf[kBufSize]; - for (;;) - { - UInt32 size = 0; - RINOK(stream->Read(buf, kBufSize, &size)) - if (size == 0) - return S_OK; - for (UInt32 i = 0; i < size; i++) - if (buf[i] != 0) - { - areThereNonZeros = true; - numZeros += i; - return S_OK; - } - numZeros += size; - if (numZeros > maxSize) - return S_OK; - } -} - } diff --git a/CPP/7zip/Archive/HandlerCont.h b/CPP/7zip/Archive/HandlerCont.h index 2dd0529..82e451a 100644 --- a/CPP/7zip/Archive/HandlerCont.h +++ b/CPP/7zip/Archive/HandlerCont.h @@ -68,9 +68,10 @@ class CHandlerImg: public IInStream, public CMyUnknownImp { - Z7_COM_UNKNOWN_IMP_3( + Z7_COM_UNKNOWN_IMP_4( IInArchive, IInArchiveGetStream, + ISequentialInStream, IInStream) Z7_COM7F_IMP(Open(IInStream *stream, const UInt64 *maxCheckStartPosition, IArchiveOpenCallback *openCallback)) @@ -80,18 +81,17 @@ class CHandlerImg: // Z7_IFACEM_IInArchive_Img(Z7_COM7F_PUREO) protected: + bool _stream_unavailData; + bool _stream_unsupportedMethod; + bool _stream_dataError; + // bool _stream_UsePackSize; + // UInt64 _stream_PackSize; UInt64 _virtPos; UInt64 _posInArc; UInt64 _size; CMyComPtr Stream; const char *_imgExt; - bool _stream_unavailData; - bool _stream_unsupportedMethod; - bool _stream_dataError; - // bool _stream_UsePackSize; - // UInt64 _stream_PackSize; - void Reset_PosInArc() { _posInArc = (UInt64)0 - 1; } void Reset_VirtPos() { _virtPos = (UInt64)0; } diff --git a/CPP/7zip/Archive/HfsHandler.cpp b/CPP/7zip/Archive/HfsHandler.cpp index 696ecd7..4049fbc 100644 --- a/CPP/7zip/Archive/HfsHandler.cpp +++ b/CPP/7zip/Archive/HfsHandler.cpp @@ -1673,8 +1673,8 @@ Z7_COM7F_IMF(CHandler::GetRawProp(UInt32 index, PROPID propID, const void **data return S_OK; } #else - UNUSED_VAR(index); - UNUSED_VAR(propID); + UNUSED_VAR(index) + UNUSED_VAR(propID) #endif return S_OK; } @@ -1786,14 +1786,14 @@ Z7_COM7F_IMF(CHandler::Close()) static const UInt32 kCompressionBlockSize = 1 << 16; -CDecoder::CDecoder() +CDecoder::CDecoder(bool IsAdlerOptional) { - _zlibDecoderSpec = new NCompress::NZlib::CDecoder(); - _zlibDecoder = _zlibDecoderSpec; - - _lzfseDecoderSpec = new NCompress::NLzfse::CDecoder(); - _lzfseDecoder = _lzfseDecoderSpec; - _lzfseDecoderSpec->LzvnMode = true; + /* Some new hfs files contain zlib resource fork without Adler checksum. + We do not know how we must detect case where there is Adler + checksum or there is no Adler checksum. + */ + _zlibDecoder->IsAdlerOptional = IsAdlerOptional; + _lzfseDecoder->LzvnMode = true; } HRESULT CDecoder::ExtractResourceFork_ZLIB( @@ -1862,8 +1862,7 @@ HRESULT CDecoder::ExtractResourceFork_ZLIB( if (prev != dataSize2) return S_FALSE; - CBufInStream *bufInStreamSpec = new CBufInStream; - CMyComPtr bufInStream = bufInStreamSpec; + CMyComPtr2_Create bufInStream; // bool padError = false; UInt64 outPos = 0; @@ -1899,11 +1898,11 @@ HRESULT CDecoder::ExtractResourceFork_ZLIB( else { const UInt64 blockSize64 = blockSize; - bufInStreamSpec->Init(buf, size); - RINOK(_zlibDecoder->Code(bufInStream, outStream, NULL, &blockSize64, NULL)) - if (_zlibDecoderSpec->GetOutputProcessedSize() != blockSize) + bufInStream->Init(buf, size); + RINOK(_zlibDecoder.Interface()->Code(bufInStream, outStream, NULL, &blockSize64, NULL)) + if (_zlibDecoder->GetOutputProcessedSize() != blockSize) return S_FALSE; - const UInt64 inSize = _zlibDecoderSpec->GetInputProcessedSize(); + const UInt64 inSize = _zlibDecoder->GetInputProcessedSize(); if (inSize != size) { if (inSize > size) @@ -2005,8 +2004,7 @@ HRESULT CDecoder::ExtractResourceFork_LZFSE( const size_t kBufSize = kCompressionBlockSize; _buf.Alloc(kBufSize + 0x10); // we need 1 additional bytes for uncompressed chunk header - CBufInStream *bufInStreamSpec = new CBufInStream; - CMyComPtr bufInStream = bufInStreamSpec; + CMyComPtr2_Create bufInStream; UInt64 outPos = 0; @@ -2042,8 +2040,8 @@ HRESULT CDecoder::ExtractResourceFork_LZFSE( { const UInt64 blockSize64 = blockSize; const UInt64 packSize64 = size; - bufInStreamSpec->Init(buf, size); - RINOK(_lzfseDecoder->Code(bufInStream, outStream, &packSize64, &blockSize64, NULL)) + bufInStream->Init(buf, size); + RINOK(_lzfseDecoder.Interface()->Code(bufInStream, outStream, &packSize64, &blockSize64, NULL)) // in/out sizes were checked in Code() } @@ -2100,8 +2098,8 @@ HRESULT CDecoder::ExtractResourceFork_ZBM( const size_t kBufSize = kCompressionBlockSize; _buf.Alloc(kBufSize + 0x10); // we need 1 additional bytes for uncompressed chunk header - CBufInStream *bufInStreamSpec = new CBufInStream; - CMyComPtr bufInStream = bufInStreamSpec; + CBufInStream *bufInStream = new CBufInStream; + CMyComPtr bufInStream = bufInStream; UInt64 outPos = 0; @@ -2222,7 +2220,7 @@ HRESULT CDecoder::ExtractResourceFork_ZBM( // const UInt64 blockSize64 = blockSize; // const UInt64 packSize64 = size; - // bufInStreamSpec->Init(buf, size); + // bufInStream->Init(buf, size); // RINOK(_zbmDecoderSpec->Code(bufInStream, outStream, &packSize64, &blockSize64, NULL)); // in/out sizes were checked in Code() } @@ -2263,24 +2261,23 @@ HRESULT CDecoder::Extract( if (compressHeader.Method == kMethod_ZLIB_ATTR || compressHeader.Method == kMethod_LZVN_ATTR) { - CBufInStream *bufInStreamSpec = new CBufInStream; - CMyComPtr bufInStream = bufInStreamSpec; + CMyComPtr2_Create bufInStream; const size_t packSize = data->Size() - compressHeader.DataPos; - bufInStreamSpec->Init(*data + compressHeader.DataPos, packSize); + bufInStream->Init(*data + compressHeader.DataPos, packSize); if (compressHeader.Method == kMethod_ZLIB_ATTR) { - const HRESULT hres = _zlibDecoder->Code(bufInStream, realOutStream, + const HRESULT hres = _zlibDecoder.Interface()->Code(bufInStream, realOutStream, NULL, &compressHeader.UnpackSize, NULL); if (hres == S_OK) - if (_zlibDecoderSpec->GetOutputProcessedSize() == compressHeader.UnpackSize - && _zlibDecoderSpec->GetInputProcessedSize() == packSize) + if (_zlibDecoder->GetOutputProcessedSize() == compressHeader.UnpackSize + && _zlibDecoder->GetInputProcessedSize() == packSize) opRes = NExtract::NOperationResult::kOK; return hres; } { const UInt64 packSize64 = packSize; - const HRESULT hres = _lzfseDecoder->Code(bufInStream, realOutStream, + const HRESULT hres = _lzfseDecoder.Interface()->Code(bufInStream, realOutStream, &packSize64, &compressHeader.UnpackSize, NULL); if (hres == S_OK) { @@ -2298,6 +2295,8 @@ HRESULT CDecoder::Extract( inStreamFork, realOutStream, forkSize, compressHeader.UnpackSize, progressStart, extractCallback); + // for debug: + // hres = NCompress::CopyStream(inStreamFork, realOutStream, NULL); } else if (compressHeader.Method == NHfs::kMethod_LZVN_RSRC) { @@ -2350,18 +2349,21 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, const size_t kBufSize = kCompressionBlockSize; CByteBuffer buf(kBufSize + 0x10); // we need 1 additional bytes for uncompressed chunk header - CDecoder decoder; + // there are hfs without adler in zlib. + CDecoder decoder(true); // IsAdlerOptional for (i = 0;; i++, currentTotalSize += currentItemSize) { RINOK(extractCallback->SetCompleted(¤tTotalSize)) - if (i == numItems) + if (i >= numItems) break; const UInt32 index = allFilesMode ? i : indices[i]; const CRef &ref = Refs[index]; const CItem &item = Items[ref.ItemIndex]; currentItemSize = Get_UnpackSize_of_Ref(ref); + int opRes; + { CMyComPtr realOutStream; const Int32 askMode = testMode ? NExtract::NAskMode::kTest : @@ -2380,7 +2382,7 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, RINOK(extractCallback->PrepareOperation(askMode)) UInt64 pos = 0; - int opRes = NExtract::NOperationResult::kDataError; + opRes = NExtract::NOperationResult::kDataError; const CFork *fork = NULL; if (ref.AttrIndex >= 0) @@ -2488,7 +2490,7 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, opRes = NExtract::NOperationResult::kDataError; } } - realOutStream.Release(); + } RINOK(extractCallback->SetOperationResult(opRes)) } return S_OK; @@ -2508,8 +2510,8 @@ HRESULT CHandler::GetForkStream(const CFork &fork, ISequentialInStream **stream) if (!fork.IsOk(Header.BlockSizeLog)) return S_FALSE; - CExtentsStream *extentStreamSpec = new CExtentsStream(); - CMyComPtr extentStream = extentStreamSpec; + CMyComPtr2 extentStream; + extentStream.Create_if_Empty(); UInt64 rem = fork.Size; UInt64 virt = 0; @@ -2531,7 +2533,7 @@ HRESULT CHandler::GetForkStream(const CFork &fork, ISequentialInStream **stream) se.Virt = virt; virt += cur; rem -= cur; - extentStreamSpec->Extents.Add(se); + extentStream->Extents.Add(se); } if (rem != 0) @@ -2540,9 +2542,9 @@ HRESULT CHandler::GetForkStream(const CFork &fork, ISequentialInStream **stream) CSeekExtent se; se.Phy = 0; se.Virt = virt; - extentStreamSpec->Extents.Add(se); - extentStreamSpec->Stream = _stream; - extentStreamSpec->Init(); + extentStream->Extents.Add(se); + extentStream->Stream = _stream; + extentStream->Init(); *stream = extentStream.Detach(); return S_OK; } diff --git a/CPP/7zip/Archive/HfsHandler.h b/CPP/7zip/Archive/HfsHandler.h index 0006e12..6d869c2 100644 --- a/CPP/7zip/Archive/HfsHandler.h +++ b/CPP/7zip/Archive/HfsHandler.h @@ -23,7 +23,7 @@ struct CCompressHeader bool IsResource; bool IsMethod_Compressed_Inline() const { return DataPos == k_decmpfs_HeaderSize; } - bool IsMethod_Uncompressed_Inline() const { return DataPos == k_decmpfs_HeaderSize + 1; } + bool IsMethod_Uncompressed_Inline() const { return DataPos == k_decmpfs_HeaderSize + 1; } bool IsMethod_Resource() const { return IsResource; } void Parse(const Byte *p, size_t size); @@ -48,11 +48,8 @@ void MethodsMaskToProp(UInt32 methodsMask, NWindows::NCOM::CPropVariant &prop); class CDecoder { - NCompress::NZlib::CDecoder *_zlibDecoderSpec; - CMyComPtr _zlibDecoder; - - NCompress::NLzfse::CDecoder *_lzfseDecoderSpec; - CMyComPtr _lzfseDecoder; + CMyComPtr2_Create _zlibDecoder; + CMyComPtr2_Create _lzfseDecoder; CByteBuffer _tableBuf; CByteBuffer _buf; @@ -82,7 +79,7 @@ class CDecoder UInt64 progressStart, IArchiveExtractCallback *extractCallback, int &opRes); - CDecoder(); + CDecoder(bool IsAdlerOptional); }; }} diff --git a/CPP/7zip/Archive/IArchive.h b/CPP/7zip/Archive/IArchive.h index 3e68ac3..a817015 100644 --- a/CPP/7zip/Archive/IArchive.h +++ b/CPP/7zip/Archive/IArchive.h @@ -565,6 +565,56 @@ Z7_IFACE_CONSTR_ARCHIVE(IArchiveKeepModeForNextOpen, 0x04) Z7_IFACE_CONSTR_ARCHIVE(IArchiveAllowTail, 0x05) +namespace NRequestMemoryUseFlags +{ + const UInt32 k_AllowedSize_WasForced = 1 << 0; // (*allowedSize) was forced by -mmemx or -smemx + const UInt32 k_DefaultLimit_Exceeded = 1 << 1; // default limit of archive format was exceeded + const UInt32 k_MLimit_Exceeded = 1 << 2; // -mmemx value was exceeded + const UInt32 k_SLimit_Exceeded = 1 << 3; // -smemx value was exceeded + + const UInt32 k_NoErrorMessage = 1 << 10; // do not show error message, and show only request + const UInt32 k_IsReport = 1 << 11; // only report is required, without user request + + const UInt32 k_SkipArc_IsExpected = 1 << 12; // NRequestMemoryAnswerFlags::k_SkipArc flag answer is expected + const UInt32 k_Report_SkipArc = 1 << 13; // report about SkipArc operation + + // const UInt32 k_SkipBigFile_IsExpected = 1 << 14; // NRequestMemoryAnswerFlags::k_SkipBigFiles flag answer is expected (unused) + // const UInt32 k_Report_SkipBigFile = 1 << 15; // report about SkipFile operation (unused) + + // const UInt32 k_SkipBigFiles_IsExpected = 1 << 16; // NRequestMemoryAnswerFlags::k_SkipBigFiles flag answer is expected (unused) + // const UInt32 k_Report_SkipBigFiles = 1 << 17; // report that all big files will be skipped (unused) +} + +namespace NRequestMemoryAnswerFlags +{ + const UInt32 k_Allow = 1 << 0; // allow further archive extraction + const UInt32 k_Stop = 1 << 1; // for exit (and return_code == E_ABORT is used) + const UInt32 k_SkipArc = 1 << 2; // skip current archive extraction + // const UInt32 k_SkipBigFile = 1 << 4; // skip extracting of files that exceed limit (unused) + // const UInt32 k_SkipBigFiles = 1 << 5; // skip extracting of files that exceed limit (unused) + const UInt32 k_Limit_Exceeded = 1 << 10; // limit was exceeded +} + +/* + *allowedSize is in/out: + in : default allowed memory usage size or forced size, if it was changed by switch -mmemx. + out : value specified by user or unchanged value. + + *answerFlags is in/out: + *answerFlags must be set by caller before calling for default action, + + indexType : must be set with NEventIndexType::* constant + (indexType == kNoIndex), if request for whole archive. + index : must be set for some (indexType) types (if + fileIndex , if (indexType == NEventIndexType::kInArcIndex) + 0, if if (indexType == kNoIndex) + path : NULL can be used for any indexType. +*/ +#define Z7_IFACEM_IArchiveRequestMemoryUseCallback(x) \ + x(RequestMemoryUse(UInt32 flags, UInt32 indexType, UInt32 index, const wchar_t *path, \ + UInt64 requiredSize, UInt64 *allowedSize, UInt32 *answerFlags)) +Z7_IFACE_CONSTR_ARCHIVE(IArchiveRequestMemoryUseCallback, 0x09) + struct CStatProp { diff --git a/CPP/7zip/Archive/Icons/zst.ico b/CPP/7zip/Archive/Icons/zst.ico new file mode 100644 index 0000000..e645e02 Binary files /dev/null and b/CPP/7zip/Archive/Icons/zst.ico differ diff --git a/CPP/7zip/Archive/IhexHandler.cpp b/CPP/7zip/Archive/IhexHandler.cpp index badb559..98ca0a3 100644 --- a/CPP/7zip/Archive/IhexHandler.cpp +++ b/CPP/7zip/Archive/IhexHandler.cpp @@ -7,14 +7,14 @@ #include "../../Common/ComTry.h" #include "../../Common/DynamicBuffer.h" #include "../../Common/IntToString.h" -#include "../../Common/MyVector.h" +#include "../../Common/StringToInt.h" #include "../../Windows/PropVariant.h" +#include "../Common/InBuffer.h" #include "../Common/ProgressUtils.h" #include "../Common/RegisterArc.h" #include "../Common/StreamUtils.h" -#include "../Common/InBuffer.h" namespace NArchive { namespace NIhex { @@ -68,6 +68,7 @@ Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) if (_needMoreInput) v |= kpv_ErrorFlags_UnexpectedEnd; if (_dataError) v |= kpv_ErrorFlags_DataError; prop = v; + break; } } prop.Detach(value); @@ -99,19 +100,12 @@ Z7_COM7F_IMF(CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val COM_TRY_END } -static inline int HexToByte(unsigned c) -{ - if (c >= '0' && c <= '9') return (int)(c - '0'); - if (c >= 'A' && c <= 'F') return (int)(c - 'A' + 10); - if (c >= 'a' && c <= 'f') return (int)(c - 'a' + 10); - return -1; -} static int Parse(const Byte *p) { - const int c1 = HexToByte(p[0]); if (c1 < 0) return -1; - const int c2 = HexToByte(p[1]); if (c2 < 0) return -1; - return (c1 << 4) | c2; + unsigned v0 = p[0]; Z7_PARSE_HEX_DIGIT(v0, return -1;) + unsigned v1 = p[1]; Z7_PARSE_HEX_DIGIT(v1, return -1;) + return (int)((v0 << 4) | v1); } #define kType_Data 0 @@ -123,7 +117,11 @@ static int Parse(const Byte *p) #define kType_MAX 5 -#define IS_LINE_DELIMITER(c) ((c) == 0 || (c) == 10 || (c) == 13) +// we don't want to read files with big number of spaces between records +// it's our limitation (out of specification): +static const unsigned k_NumSpaces_LIMIT = 16 + 1; + +#define IS_LINE_DELIMITER(c) (/* (c) == 0 || */ (c) == 10 || (c) == 13) API_FUNC_static_IsArc IsArc_Ihex(const Byte *p, size_t size) { @@ -141,11 +139,11 @@ API_FUNC_static_IsArc IsArc_Ihex(const Byte *p, size_t size) if (size < 4 * 2) return k_IsArc_Res_NEED_MORE; - int num = Parse(p); + const int num = Parse(p); if (num < 0) return k_IsArc_Res_NO; - int type = Parse(p + 6); + const int type = Parse(p + 6); if (type < 0 || type > kType_MAX) return k_IsArc_Res_NO; @@ -162,7 +160,7 @@ API_FUNC_static_IsArc IsArc_Ihex(const Byte *p, size_t size) sum += (unsigned)v; } - if ((sum & 0xFF) != 0) + if (sum & 0xFF) return k_IsArc_Res_NO; if (type == kType_Data) @@ -199,17 +197,17 @@ API_FUNC_static_IsArc IsArc_Ihex(const Byte *p, size_t size) p += numChars; size -= numChars; + unsigned numSpaces = k_NumSpaces_LIMIT; for (;;) { if (size == 0) return k_IsArc_Res_NEED_MORE; const Byte b = *p++; size--; - if (IS_LINE_DELIMITER(b)) - continue; if (b == ':') break; - return k_IsArc_Res_NO; + if (--numSpaces == 0 || !IS_LINE_DELIMITER(b)) + return k_IsArc_Res_NO; } } @@ -217,7 +215,7 @@ API_FUNC_static_IsArc IsArc_Ihex(const Byte *p, size_t size) } } -Z7_COM7F_IMF(CHandler::Open(IInStream *stream, const UInt64 *, IArchiveOpenCallback *)) +Z7_COM7F_IMF(CHandler::Open(IInStream *stream, const UInt64 *, IArchiveOpenCallback *openCallback)) { COM_TRY_BEGIN { @@ -229,7 +227,7 @@ Z7_COM7F_IMF(CHandler::Open(IInStream *stream, const UInt64 *, IArchiveOpenCallb { size_t size = kStartSize; RINOK(ReadStream(stream, temp, &size)) - UInt32 isArcRes = IsArc_Ihex(temp, size); + const UInt32 isArcRes = IsArc_Ihex(temp, size); if (isArcRes == k_IsArc_Res_NO) return S_FALSE; if (isArcRes == k_IsArc_Res_NEED_MORE && size != kStartSize) @@ -243,7 +241,6 @@ Z7_COM7F_IMF(CHandler::Open(IInStream *stream, const UInt64 *, IArchiveOpenCallb return E_OUTOFMEMORY; s.SetStream(stream); s.Init(); - { Byte b; if (!s.ReadByte(b)) @@ -259,6 +256,8 @@ Z7_COM7F_IMF(CHandler::Open(IInStream *stream, const UInt64 *, IArchiveOpenCallb } UInt32 globalOffset = 0; + const UInt32 k_progressStep = 1 << 24; + UInt64 progressNext = k_progressStep; for (;;) { @@ -273,16 +272,14 @@ Z7_COM7F_IMF(CHandler::Open(IInStream *stream, const UInt64 *, IArchiveOpenCallb _dataError = true; return S_FALSE; } - { - size_t numPairs = ((unsigned)num + 4); - size_t numBytes = numPairs * 2; + const size_t numPairs = (unsigned)num + 4; + const size_t numBytes = numPairs * 2; if (s.ReadBytes(temp, numBytes) != numBytes) { _needMoreInput = true; return S_FALSE; } - unsigned sum = (unsigned)num; for (size_t i = 0; i < numPairs; i++) { @@ -295,21 +292,21 @@ Z7_COM7F_IMF(CHandler::Open(IInStream *stream, const UInt64 *, IArchiveOpenCallb temp[i] = (Byte)a; sum += (unsigned)a; } - if ((sum & 0xFF) != 0) + if (sum & 0xFF) { _dataError = true; return S_FALSE; } } - unsigned type = temp[2]; + const unsigned type = temp[2]; if (type > kType_MAX) { _dataError = true; return S_FALSE; } - UInt32 a = GetBe16(temp); + const UInt32 a = GetBe16(temp); if (type == kType_Data) { @@ -322,7 +319,7 @@ Z7_COM7F_IMF(CHandler::Open(IInStream *stream, const UInt64 *, IArchiveOpenCallb } // if (num != 0) { - UInt32 offs = globalOffset + a; + const UInt32 offs = globalOffset + a; CBlock *block = NULL; if (!_blocks.IsEmpty()) { @@ -338,10 +335,21 @@ Z7_COM7F_IMF(CHandler::Open(IInStream *stream, const UInt64 *, IArchiveOpenCallb block->Data.AddData(temp + 3, (unsigned)num); } } - else if (type == kType_Eof) + else { - _phySize = s.GetProcessedSize(); + if (a != 0) // from description: the address field is typically 0. { + _dataError = true; + return S_FALSE; + } + if (type == kType_Eof) + { + if (num != 0) + { + _dataError = true; + return S_FALSE; + } + _phySize = s.GetProcessedSize(); Byte b; if (s.ReadByte(b)) { @@ -357,16 +365,9 @@ Z7_COM7F_IMF(CHandler::Open(IInStream *stream, const UInt64 *, IArchiveOpenCallb } } } + return S_OK; } - return S_OK; - } - else - { - if (a != 0) - { - _dataError = true; - return S_FALSE; - } + if (type == kType_Seg || type == kType_High) { if (num != 2) @@ -374,8 +375,8 @@ Z7_COM7F_IMF(CHandler::Open(IInStream *stream, const UInt64 *, IArchiveOpenCallb _dataError = true; return S_FALSE; } - UInt32 d = GetBe16(temp + 3); - globalOffset = d << (type == kType_Seg ? 4 : 16); + // here we use optimization trick for num shift calculation: (type == kType_Seg ? 4 : 16) + globalOffset = (UInt32)GetBe16(temp + 3) << (1u << type); } else { @@ -387,6 +388,18 @@ Z7_COM7F_IMF(CHandler::Open(IInStream *stream, const UInt64 *, IArchiveOpenCallb } } + if (openCallback) + { + const UInt64 processed = s.GetProcessedSize(); + if (processed >= progressNext) + { + progressNext = processed + k_progressStep; + const UInt64 numFiles = _blocks.Size(); + RINOK(openCallback->SetCompleted(&numFiles, &processed)) + } + } + + unsigned numSpaces = k_NumSpaces_LIMIT; for (;;) { Byte b; @@ -395,12 +408,13 @@ Z7_COM7F_IMF(CHandler::Open(IInStream *stream, const UInt64 *, IArchiveOpenCallb _needMoreInput = true; return S_FALSE; } - if (IS_LINE_DELIMITER(b)) - continue; if (b == ':') break; - _dataError = true; - return S_FALSE; + if (--numSpaces == 0 || !IS_LINE_DELIMITER(b)) + { + _dataError = true; + return S_FALSE; + } } } } @@ -409,14 +423,13 @@ Z7_COM7F_IMF(CHandler::Open(IInStream *stream, const UInt64 *, IArchiveOpenCallb COM_TRY_END } + Z7_COM7F_IMF(CHandler::Close()) { _phySize = 0; - _isArc = false; _needMoreInput = false; _dataError = false; - _blocks.Clear(); return S_OK; } @@ -436,53 +449,40 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, UInt32 i; for (i = 0; i < numItems; i++) totalSize += _blocks[allFilesMode ? i : indices[i]].Data.GetPos(); - extractCallback->SetTotal(totalSize); - - UInt64 currentTotalSize = 0; - UInt64 currentItemSize; + RINOK(extractCallback->SetTotal(totalSize)) - CLocalProgress *lps = new CLocalProgress; - CMyComPtr progress = lps; + CMyComPtr2_Create lps; lps->Init(extractCallback, false); - for (i = 0; i < numItems; i++, currentTotalSize += currentItemSize) + for (i = 0;; i++) { - currentItemSize = 0; - lps->InSize = lps->OutSize = currentTotalSize; + lps->InSize = lps->OutSize; RINOK(lps->SetCur()) - + if (i >= numItems) + break; const UInt32 index = allFilesMode ? i : indices[i]; const CByteDynamicBuffer &data = _blocks[index].Data; - currentItemSize = data.GetPos(); - - CMyComPtr realOutStream; - const Int32 askMode = testMode ? - NExtract::NAskMode::kTest : - NExtract::NAskMode::kExtract; - - RINOK(extractCallback->GetStream(index, &realOutStream, askMode)) - - if (!testMode && !realOutStream) - continue; - - extractCallback->PrepareOperation(askMode); - - if (realOutStream) + lps->OutSize += data.GetPos(); { - RINOK(WriteStream(realOutStream, (const Byte *)data, data.GetPos())) + CMyComPtr realOutStream; + const Int32 askMode = testMode ? + NExtract::NAskMode::kTest : + NExtract::NAskMode::kExtract; + RINOK(extractCallback->GetStream(index, &realOutStream, askMode)) + if (!testMode && !realOutStream) + continue; + RINOK(extractCallback->PrepareOperation(askMode)) + if (realOutStream) + RINOK(WriteStream(realOutStream, (const Byte *)data, data.GetPos())) } - - realOutStream.Release(); RINOK(extractCallback->SetOperationResult(NExtract::NOperationResult::kOK)) } - lps->InSize = lps->OutSize = currentTotalSize; - return lps->SetCur(); - + return S_OK; COM_TRY_END } -// k_Signature: { ':', '1' } +// k_Signature: { ':' } REGISTER_ARC_I_NO_SIG( "IHex", "ihex", NULL, 0xCD, diff --git a/CPP/7zip/Archive/Iso/IsoHandler.cpp b/CPP/7zip/Archive/Iso/IsoHandler.cpp index 0c63c71..c085386 100644 --- a/CPP/7zip/Archive/Iso/IsoHandler.cpp +++ b/CPP/7zip/Archive/Iso/IsoHandler.cpp @@ -85,13 +85,21 @@ static void AddString(AString &s, const char *name, const Byte *p, unsigned size { AString d; d.SetFrom((const char *)p, i); - s += '\n'; s += name; s += ": "; s += d; + s.Add_LF(); } } +static void AddProp_Size64(AString &s, const char *name, UInt64 size) +{ + s += name; + s += ": "; + s.Add_UInt64(size); + s.Add_LF(); +} + #define ADD_STRING(n, v) AddString(s, n, vol. v, sizeof(vol. v)) static void AddErrorMessage(AString &s, const char *message) @@ -122,6 +130,11 @@ Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) ADD_STRING("Copyright", CopyrightFileId); ADD_STRING("Abstract", AbstractFileId); ADD_STRING("Bib", BibFileId); + // ADD_STRING("EscapeSequence", EscapeSequence); + AddProp_Size64(s, "VolumeSpaceSize", vol.Get_VolumeSpaceSize_inBytes()); + AddProp_Size64(s, "VolumeSetSize", vol.VolumeSetSize); + AddProp_Size64(s, "VolumeSequenceNumber", vol.VolumeSequenceNumber); + prop = s; break; } @@ -328,27 +341,26 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, else totalSize += _archive.GetBootItemSize(index - _archive.Refs.Size()); } - extractCallback->SetTotal(totalSize); + RINOK(extractCallback->SetTotal(totalSize)) UInt64 currentTotalSize = 0; UInt64 currentItemSize; - NCompress::CCopyCoder *copyCoderSpec = new NCompress::CCopyCoder(); - CMyComPtr copyCoder = copyCoderSpec; - - CLocalProgress *lps = new CLocalProgress; - CMyComPtr progress = lps; + CMyComPtr2_Create lps; lps->Init(extractCallback, false); + CMyComPtr2_Create copyCoder; + CMyComPtr2_Create inStream; + inStream->SetStream(_stream); - CLimitedSequentialInStream *streamSpec = new CLimitedSequentialInStream; - CMyComPtr inStream(streamSpec); - streamSpec->SetStream(_stream); - - for (i = 0; i < numItems; i++, currentTotalSize += currentItemSize) + for (i = 0;; i++, currentTotalSize += currentItemSize) { lps->InSize = lps->OutSize = currentTotalSize; RINOK(lps->SetCur()) + if (i >= numItems) + break; currentItemSize = 0; + Int32 opRes = NExtract::NOperationResult::kOK; + { CMyComPtr realOutStream; const Int32 askMode = testMode ? NExtract::NAskMode::kTest : @@ -385,7 +397,6 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, RINOK(extractCallback->PrepareOperation(askMode)) - bool isOK = true; if (index < (UInt32)_archive.Refs.Size()) { const CRef &ref = _archive.Refs[index]; @@ -397,11 +408,11 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, continue; lps->InSize = lps->OutSize = currentTotalSize + offset; RINOK(InStream_SeekSet(_stream, (UInt64)item2.ExtentLocation * kBlockSize)) - streamSpec->Init(item2.Size); - RINOK(copyCoder->Code(inStream, realOutStream, NULL, NULL, progress)) - if (copyCoderSpec->TotalSize != item2.Size) + inStream->Init(item2.Size); + RINOK(copyCoder.Interface()->Code(inStream, realOutStream, NULL, NULL, lps)) + if (copyCoder->TotalSize != item2.Size) { - isOK = false; + opRes = NExtract::NOperationResult::kDataError; break; } offset += item2.Size; @@ -410,15 +421,14 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, else { RINOK(InStream_SeekSet(_stream, (UInt64)blockIndex * kBlockSize)) - streamSpec->Init(currentItemSize); - RINOK(copyCoder->Code(inStream, realOutStream, NULL, NULL, progress)) - if (copyCoderSpec->TotalSize != currentItemSize) - isOK = false; + inStream->Init(currentItemSize); + RINOK(copyCoder.Interface()->Code(inStream, realOutStream, NULL, NULL, lps)) + if (copyCoder->TotalSize != currentItemSize) + opRes = NExtract::NOperationResult::kDataError; } - realOutStream.Release(); - RINOK(extractCallback->SetOperationResult(isOK ? - NExtract::NOperationResult::kOK: - NExtract::NOperationResult::kDataError)) + // realOutStream.Release(); + } + RINOK(extractCallback->SetOperationResult(opRes)) } return S_OK; COM_TRY_END diff --git a/CPP/7zip/Archive/Iso/IsoIn.cpp b/CPP/7zip/Archive/Iso/IsoIn.cpp index 1d3a42f..0054abf 100644 --- a/CPP/7zip/Archive/Iso/IsoIn.cpp +++ b/CPP/7zip/Archive/Iso/IsoIn.cpp @@ -287,11 +287,17 @@ void CInArchive::ReadVolumeDescriptor(CVolumeDescriptor &d) ReadDateTime(d.MTime); ReadDateTime(d.ExpirationTime); ReadDateTime(d.EffectiveTime); - d.FileStructureVersion = ReadByte(); // = 1 - SkipZeros(1); + const Byte fileStructureVersion = ReadByte(); + // d.FileStructureVersion = fileStructureVersion; + if (fileStructureVersion != 1 && // ECMA-119 + fileStructureVersion != 2) // some ISO files have fileStructureVersion == 2. + { + // v24.05: we ignore that field, because we don't know what exact values are allowed there + // throw CHeaderErrorException(); + } + SkipZeros(1); // (Reserved for future standardization) ReadBytes(d.ApplicationUse, sizeof(d.ApplicationUse)); - - // Most ISO contains zeros in the following field (reserved for future standardization). + // Most ISO contain zeros in the following field (reserved for future standardization). // But some ISO programs write some data to that area. // So we disable check for zeros. Skip(653); // SkipZeros(653); @@ -595,7 +601,16 @@ HRESULT CInArchive::Open2() const CVolumeDescriptor &vd = VolDescs[MainVolDescIndex]; if (vd.LogicalBlockSize != kBlockSize) return S_FALSE; - + + { + FOR_VECTOR (i, VolDescs) + { + const CVolumeDescriptor &vd2 = VolDescs[i]; + UpdatePhySize(0, vd2.Get_VolumeSpaceSize_inBytes()); + } + } + + IsArc = true; (CDirRecord &)_rootDir = vd.RootDirRecord; @@ -615,6 +630,21 @@ HRESULT CInArchive::Open2() } } } + + { + // find boot item for expand: + // UEFI Specification : 13.3.2.1. ISO-9660 and El Torito + _expand_BootEntries_index = -1; + FOR_VECTOR (i, BootEntries) + { + const CBootInitialEntry &be = BootEntries[i]; + if (be.SectorCount <= 1 && be.BootMediaType == NBootMediaType::kNoEmulation) + if (_expand_BootEntries_index == -1 + || be.LoadRBA >= BootEntries[_expand_BootEntries_index].LoadRBA) + _expand_BootEntries_index = (int)i; + } + } + { FOR_VECTOR (i, BootEntries) { @@ -670,6 +700,8 @@ void CInArchive::Clear() BootEntries.Clear(); SuspSkipSize = 0; IsSusp = false; + + _expand_BootEntries_index = -1; } @@ -684,7 +716,17 @@ UInt64 CInArchive::GetBootItemSize(unsigned index) const if (startPos < _fileSize) { const UInt64 rem = _fileSize - startPos; - if (rem < size) + /* + UEFI modification to ISO specification: + because SectorCount is 16-bit, size is limited by (32 MB). + UEFI Specification : + 13.3.2.1. ISO-9660 and El Torito + If the value of Sector Count is set to 0 or 1, + EFI will assume the system partition consumes the space + from the beginning of the "no emulation" image to the end of the CD-ROM. + */ + // + if ((int)index == _expand_BootEntries_index || rem < size) size = rem; } return size; diff --git a/CPP/7zip/Archive/Iso/IsoIn.h b/CPP/7zip/Archive/Iso/IsoIn.h index f3e4751..e7d1fee 100644 --- a/CPP/7zip/Archive/Iso/IsoIn.h +++ b/CPP/7zip/Archive/Iso/IsoIn.h @@ -214,17 +214,19 @@ struct CVolumeDescriptor CDateTime MTime; CDateTime ExpirationTime; CDateTime EffectiveTime; - Byte FileStructureVersion; // = 1; + // Byte FileStructureVersion; // = 1; Byte ApplicationUse[512]; bool IsJoliet() const { if ((VolFlags & 1) != 0) return false; - Byte b = EscapeSequence[2]; + const Byte b = EscapeSequence[2]; return (EscapeSequence[0] == 0x25 && EscapeSequence[1] == 0x2F && (b == 0x40 || b == 0x43 || b == 0x45)); } + + UInt64 Get_VolumeSpaceSize_inBytes() const { return (UInt64)VolumeSpaceSize * LogicalBlockSize; } }; struct CRef @@ -293,6 +295,8 @@ class CInArchive bool IsSusp; unsigned SuspSkipSize; + int _expand_BootEntries_index; + CRecordVector UniqStartLocations; void UpdatePhySize(const UInt32 blockIndex, const UInt64 size) diff --git a/CPP/7zip/Archive/Iso/IsoItem.h b/CPP/7zip/Archive/Iso/IsoItem.h index 9556805..65829d0 100644 --- a/CPP/7zip/Archive/Iso/IsoItem.h +++ b/CPP/7zip/Archive/Iso/IsoItem.h @@ -179,7 +179,7 @@ struct CDirRecord if (flags & (1 << 1)) link += "./"; else if (flags & (1 << 2)) link += "../"; - else if (flags & (1 << 3)) link += '/'; + else if (flags & (1 << 3)) link.Add_Slash(); else needSlash = true; @@ -201,7 +201,7 @@ struct CDirRecord break; if (needSlash) - link += '/'; + link.Add_Slash(); } return true; diff --git a/CPP/7zip/Archive/LvmHandler.cpp b/CPP/7zip/Archive/LvmHandler.cpp new file mode 100644 index 0000000..d144b2a --- /dev/null +++ b/CPP/7zip/Archive/LvmHandler.cpp @@ -0,0 +1,1107 @@ +// LvmHandler.cpp + +#include "StdAfx.h" + +#include "../../../C/7zCrc.h" +#include "../../../C/CpuArch.h" + +#include "../../Common/ComTry.h" +#include "../../Common/MyBuffer.h" +#include "../../Common/StringToInt.h" + +#include "../../Windows/PropVariantUtils.h" +#include "../../Windows/TimeUtils.h" + +#include "../Common/RegisterArc.h" +#include "../Common/StreamUtils.h" + +#include "HandlerCont.h" + +#define Get32(p) GetUi32(p) +#define Get64(p) GetUi64(p) + +#define LE_32(offs, dest) dest = Get32(p + (offs)) +#define LE_64(offs, dest) dest = Get64(p + (offs)) + +using namespace NWindows; + +namespace NArchive { +namespace NLvm { + +#define SIGNATURE { 'L', 'A', 'B', 'E', 'L', 'O', 'N', 'E' } + +static const unsigned k_SignatureSize = 8; +static const Byte k_Signature[k_SignatureSize] = SIGNATURE; + +static const unsigned k_Signature2Size = 8; +static const Byte k_Signature2[k_Signature2Size] = + { 'L', 'V', 'M', '2', ' ', '0', '0', '1' }; + +static const Byte FMTT_MAGIC[16] = + { ' ', 'L', 'V', 'M', '2', ' ', 'x', '[', '5', 'A', '%', 'r', '0', 'N', '*', '>' }; + +static const UInt32 kSectorSize = 512; + + +struct CPropVal +{ + bool IsNumber; + AString String; + UInt64 Number; + + CPropVal(): IsNumber(false), Number(0) {} +}; + + +struct CConfigProp +{ + AString Name; + + bool IsVector; + CPropVal Val; + CObjectVector Vector; + + CConfigProp(): IsVector(false) {} +}; + + +class CConfigItem +{ +public: + AString Name; + CObjectVector Props; + CObjectVector Items; + + const char *ParseItem(const char *s, int numAllowedLevels); + + int FindProp(const char *name) const throw(); + bool GetPropVal_Number(const char *name, UInt64 &val) const throw(); + bool GetPropVal_String(const char *name, AString &val) const; + + int FindSubItem(const char *tag) const throw(); +}; + +struct CConfig +{ + CConfigItem Root; + + bool Parse(const char *s); +}; + + +static bool IsSpaceChar(char c) +{ + return (c == ' ' || c == '\t' || c == 0x0D || c == 0x0A); +} + +static const char *SkipSpaces(const char * s) +{ + for (;; s++) + { + const char c = *s; + if (c == 0) + return s; + if (!IsSpaceChar(c)) + { + if (c != '#') + return s; + s++; + for (;;) + { + const char c2 = *s; + if (c2 == 0) + return s; + if (c2 == '\n') + break; + s++; + } + } + } +} + +#define SKIP_SPACES(s) s = SkipSpaces(s); + +int CConfigItem::FindProp(const char *name) const throw() +{ + FOR_VECTOR (i, Props) + if (Props[i].Name == name) + return (int)i; + return -1; +} + +bool CConfigItem::GetPropVal_Number(const char *name, UInt64 &val) const throw() +{ + val = 0; + int index = FindProp(name); + if (index < 0) + return false; + const CConfigProp &prop = Props[index]; + if (prop.IsVector) + return false; + if (!prop.Val.IsNumber) + return false; + val = prop.Val.Number; + return true; +} + +bool CConfigItem::GetPropVal_String(const char *name, AString &val) const +{ + val.Empty(); + int index = FindProp(name); + if (index < 0) + return false; + const CConfigProp &prop = Props[index]; + if (prop.IsVector) + return false; + if (prop.Val.IsNumber) + return false; + val = prop.Val.String; + return true; +} + +int CConfigItem::FindSubItem(const char *tag) const throw() +{ + FOR_VECTOR (i, Items) + if (Items[i].Name == tag) + return (int)i; + return -1; +} + +static const char *FillProp(const char *s, CPropVal &val) +{ + SKIP_SPACES(s) + const char c = *s; + if (c == 0) + return NULL; + + if (c == '\"') + { + s++; + val.IsNumber = false; + val.String.Empty(); + + for (;;) + { + const char c2 = *s; + if (c2 == 0) + return NULL; + s++; + if (c2 == '\"') + break; + val.String += c2; + } + } + else + { + const char *end; + val.IsNumber = true; + val.Number = ConvertStringToUInt64(s, &end); + if (s == end) + return NULL; + s = end; + } + + SKIP_SPACES(s) + return s; +} + + +const char *CConfigItem::ParseItem(const char *s, int numAllowedLevels) +{ + if (numAllowedLevels < 0) + return NULL; + + for (;;) + { + SKIP_SPACES(s) + const char *beg = s; + + for (;; s++) + { + char c = *s; + if (c == 0 || c == '}') + { + if (s != beg) + return NULL; + return s; + } + if (IsSpaceChar(c) || c == '=' || c == '{') + break; + } + + if (s == beg) + return NULL; + + AString name; + name.SetFrom(beg, (unsigned)(s - beg)); + + SKIP_SPACES(s) + + if (*s == 0 || *s == '}') + return NULL; + + if (*s == '{') + { + s++; + CConfigItem &item = Items.AddNew(); + item.Name = name; + s = item.ParseItem(s, numAllowedLevels - 1); + if (!s) + return NULL; + if (*s != '}') + return NULL; + s++; + continue; + } + + if (*s != '=') + continue; + + s++; + SKIP_SPACES(s) + if (*s == 0) + return NULL; + CConfigProp &prop = Props.AddNew(); + + prop.Name = name; + + if (*s == '[') + { + s++; + prop.IsVector = true; + + for (;;) + { + SKIP_SPACES(s) + char c = *s; + if (c == 0) + return NULL; + if (c == ']') + { + s++; + break; + } + + CPropVal val; + + s = FillProp(s, val); + if (!s) + return NULL; + prop.Vector.Add(val); + SKIP_SPACES(s) + + if (*s == ',') + { + s++; + continue; + } + if (*s != ']') + return NULL; + s++; + break; + } + } + else + { + prop.IsVector = false; + s = FillProp(s, prop.Val); + if (!s) + return NULL; + } + } +} + + +bool CConfig::Parse(const char *s) +{ + s = Root.ParseItem(s, 10); + if (!s) + return false; + SKIP_SPACES(s) + return *s == 0; +} + + +/* +static const CUInt32PCharPair g_PartitionFlags[] = +{ + { 0, "Sys" }, + { 1, "Ignore" }, + { 2, "Legacy" }, + { 60, "Win-Read-only" }, + { 62, "Win-Hidden" }, + { 63, "Win-Not-Automount" } +}; +*/ + +/* +static inline char GetHex(unsigned t) { return (char)(((t < 10) ? ('0' + t) : ('A' + (t - 10)))); } + +static void PrintHex(unsigned v, char *s) +{ + s[0] = GetHex((v >> 4) & 0xF); + s[1] = GetHex(v & 0xF); +} + +static void ConvertUInt16ToHex4Digits(UInt32 val, char *s) throw() +{ + PrintHex(val >> 8, s); + PrintHex(val & 0xFF, s + 2); +} + +static void GuidToString(const Byte *g, char *s) +{ + ConvertUInt32ToHex8Digits(Get32(g ), s); s += 8; *s++ = '-'; + ConvertUInt16ToHex4Digits(Get16(g + 4), s); s += 4; *s++ = '-'; + ConvertUInt16ToHex4Digits(Get16(g + 6), s); s += 4; *s++ = '-'; + for (unsigned i = 0; i < 8; i++) + { + if (i == 2) + *s++ = '-'; + PrintHex(g[8 + i], s); + s += 2; + } + *s = 0; +} + +*/ + +struct CPhyVol +{ + AString Name; + + // AString id; + // AString device; // "/dev/sda2" + // AString status; // ["ALLOCATABLE"] + // AString flags; // [] + // UInt64 dev_size; // in sectors + UInt64 pe_start; // in sectors + UInt64 pe_count; // in extents + + bool Parse(const CConfigItem &ci) + { + Name = ci.Name; + // ci.GetPropVal_String("id", id); + // ci.GetPropVal_String("device", device); + bool res = true; + // if (!ci.GetPropVal_Number("dev_size", dev_size)) res = false; + if (!ci.GetPropVal_Number("pe_start", pe_start)) res = false; + if (!ci.GetPropVal_Number("pe_count", pe_count)) res = false; + return res; + } +}; + +struct CStripe +{ + AString Name; // "pv0"; + UInt64 ExtentOffset; // ???? +}; + +struct CSegment +{ + UInt64 start_extent; + UInt64 extent_count; + AString type; + CObjectVector stripes; + + bool IsPosSizeOk() const + { + return + (start_extent < ((UInt64)1 << 63)) && + (extent_count < ((UInt64)1 << 63)); + } + + UInt64 GetEndExtent() const { return start_extent + extent_count; } + + bool Parse(const CConfigItem &si) + { + UInt64 stripe_count; + + if (!si.GetPropVal_Number("start_extent", start_extent)) return false; + if (!si.GetPropVal_Number("extent_count", extent_count)) return false; + if (!si.GetPropVal_Number("stripe_count", stripe_count)) return false; + if (!si.GetPropVal_String("type", type)) return false; + + //if (stripe_count != 1) return false; + + const int spi = si.FindProp("stripes"); + if (spi < 0) + return false; + + const CConfigProp &prop = si.Props[spi]; + if (!prop.IsVector) + return false; + + if (stripe_count > (1 << 20)) + return false; + + const unsigned numStripes = (unsigned)stripe_count; + if (prop.Vector.Size() != numStripes * 2) + return false; + + for (unsigned i = 0; i < numStripes; i++) + { + const CPropVal &v0 = prop.Vector[i * 2]; + const CPropVal &v1 = prop.Vector[i * 2 + 1]; + if (v0.IsNumber || !v1.IsNumber) + return false; + CStripe stripe; + stripe.Name = v0.String; + stripe.ExtentOffset = v1.Number; + stripes.Add(stripe); + } + + return true; + } +}; + + +struct CLogVol +{ + bool IsSupported; + + AString Name; + + AString id; + AString status; // ["READ", "WRITE", "VISIBLE"] + AString flags; // [] + + // UInt64 Pos; + // UInt64 Size; + + // UInt64 GetSize() const { return Size; } + // UInt64 GetPos() const { return Pos; } + + CObjectVector Segments; + + CLogVol(): /* Pos(0), Size(0), */ IsSupported(false) {} + + bool Parse(const CConfigItem &ci) + { + Name = ci.Name; + + UInt64 segment_count; + if (!ci.GetPropVal_Number("segment_count", segment_count)) + return false; + + if (ci.Items.Size() != segment_count) + return false; + + FOR_VECTOR (segIndex, ci.Items) + { + const CConfigItem &si = ci.Items[segIndex]; + { + AString t ("segment"); + t.Add_UInt32(segIndex + 1); + if (si.Name != t) + return false; + } + + CSegment segment; + + if (!segment.Parse(si)) + return false; + + // item.Size += (segment.extent_count * _extentSize) << 9; + + Segments.Add(segment); + } + + IsSupported = true; + return true; + } + + bool GetNumExtents(UInt64 &numExtents) const + { + numExtents = 0; + if (Segments.IsEmpty()) + return true; + unsigned i; + for (i = 1; i < Segments.Size(); i++) + if (!Segments[i].IsPosSizeOk()) + return false; + for (i = 1; i < Segments.Size(); i++) + if (Segments[i - 1].GetEndExtent() != Segments[i].start_extent) + return false; + numExtents = Segments.Back().GetEndExtent(); + return true; + } +}; + + +struct CItem +{ + int LogVol; + int PhyVol; + UInt64 Pos; + UInt64 Size; + AString Name; + bool IsSupported; + + CItem(): LogVol(-1), PhyVol(-1), Pos(0), Size(0), IsSupported(false) {} +}; + + +struct CVolGroup +{ + CObjectVector _logVols; + CObjectVector _phyVols; + AString _id; + int _extentSizeBits; + + /* + UInt64 secno; // 3 + AString status; // ["RESIZEABLE", "READ", "WRITE"] + AString flags; // [] + UInt64 max_lv; // 0 + UInt64 max_pv; // 0 + UInt64 metadata_copies; // 0 + */ + + void Clear() + { + _logVols.Clear(); + _phyVols.Clear(); + _id.Empty(); + _extentSizeBits = -1; + } +}; + + +Z7_class_CHandler_final: public CHandlerCont, public CVolGroup +{ + Z7_IFACE_COM7_IMP(IInArchive_Cont) + + CObjectVector _items; + + UInt64 _cTime; + + bool _isArc; + + UInt64 _phySize; + CByteBuffer _buffer; + + UInt64 _cfgPos; + UInt64 _cfgSize; + + HRESULT Open2(IInStream *stream); + + virtual int GetItem_ExtractInfo(UInt32 index, UInt64 &pos, UInt64 &size) const Z7_override + { + if (index >= _items.Size()) + { + pos = _cfgPos; + size = _cfgSize; + return NExtract::NOperationResult::kOK; + } + const CItem &item = _items[index]; + if (!item.IsSupported) + return NExtract::NOperationResult::kUnsupportedMethod; + pos = item.Pos; + size = item.Size; + return NExtract::NOperationResult::kOK; + } +}; + +static const UInt32 LVM_CRC_INIT_VAL = 0xf597a6cf; + +static UInt32 Z7_FASTCALL LvmCrcCalc(const void *data, size_t size) +{ + return CrcUpdate(LVM_CRC_INIT_VAL, data, size); +} + +struct CRawLocn +{ + UInt64 Offset; /* Offset in bytes to start sector */ + UInt64 Size; /* Bytes */ + UInt32 Checksum; + UInt32 Flags; + + bool IsEmpty() const { return Offset == 0 && Size == 0; } + + void Parse(const Byte *p) + { + LE_64(0x00, Offset); + LE_64(0x08, Size); + LE_32(0x10, Checksum); + LE_32(0x14, Flags); + } +}; + +// #define MDA_HEADER_SIZE 512 + +struct mda_header +{ + UInt64 Start; /* Absolute start byte of mda_header */ + UInt64 Size; /* Size of metadata area */ + + CRecordVector raw_locns; + + bool Parse(const Byte *p, size_t size) + { + if (memcmp(p + 4, FMTT_MAGIC, 16) != 0) + return false; + UInt32 version; + LE_32(0x14, version); + if (version != 1) + return false; + LE_64(0x18, Start); + LE_64(0x20, Size); + + unsigned pos = 0x28; + + for (;;) + { + if (pos + 0x18 > size) + return false; + CRawLocn locn; + locn.Parse(p + pos); + if (locn.IsEmpty()) + break; + pos += 0x18; + raw_locns.Add(locn); + } + + return true; + } +}; + + +static int inline GetLog(UInt64 num) +{ + for (unsigned i = 0; i < 64; i++) + if (((UInt64)1 << i) == num) + return (int)i; + return -1; +} + +#define ID_LEN 32 + +HRESULT CHandler::Open2(IInStream *stream) +{ + _buffer.Alloc(kSectorSize * 2); + RINOK(ReadStream_FALSE(stream, _buffer, kSectorSize * 2)) + + const Byte *buf = _buffer; + + buf += kSectorSize; + + // label_header + + if (memcmp(buf, k_Signature, k_SignatureSize) != 0) + return S_FALSE; + const UInt64 sectorNumber = Get64(buf + 8); + if (sectorNumber != 1) + return S_FALSE; + if (Get32(buf + 16) != LvmCrcCalc(buf + 20, kSectorSize - 20)) + return S_FALSE; + + const UInt32 offsetToCont = Get32(buf + 20); + if (memcmp(buf + 24, k_Signature2, k_Signature2Size) != 0) + return S_FALSE; + + if (offsetToCont != 32) + return S_FALSE; + + // pv_header + + size_t pos = offsetToCont; + const Byte *p = buf; + + /* + { + Byte id[ID_LEN]; + memcpy(id, p + pos, ID_LEN); + } + */ + + pos += ID_LEN; + const UInt64 device_size_xl = Get64(p + pos); + pos += 8; + + _phySize = device_size_xl; + _isArc = true; + + for (;;) + { + if (pos > kSectorSize - 16) + return S_FALSE; + // disk_locn (data areas) + UInt64 offset = Get64(p + pos); + UInt64 size = Get64(p + pos + 8); + pos += 16; + if (offset == 0 && size == 0) + break; + } + + CConfig cfg; + // bool isFinded = false; + + // for (;;) + { + if (pos > kSectorSize - 16) + return S_FALSE; + // disk_locn (metadata area headers) + UInt64 offset = Get64(p + pos); + UInt64 size = Get64(p + pos + 8); + pos += 16; + if (offset == 0 && size == 0) + { + // break; + return S_FALSE; + } + + CByteBuffer meta; + const size_t sizeT = (size_t)size; + if (sizeT != size) + return S_FALSE; + meta.Alloc(sizeT); + RINOK(InStream_SeekSet(stream, offset)) + RINOK(ReadStream_FALSE(stream, meta, sizeT)) + if (Get32(meta) != LvmCrcCalc(meta + 4, kSectorSize - 4)) + return S_FALSE; + mda_header mh; + if (!mh.Parse(meta, kSectorSize)) + return S_FALSE; + + if (mh.raw_locns.Size() != 1) + return S_FALSE; + unsigned g = 0; + // for (unsigned g = 0; g < mh.raw_locns.Size(); g++) + { + const CRawLocn &locn = mh.raw_locns[g]; + + CByteBuffer vgBuf; + if (locn.Size > ((UInt32)1 << 24)) + return S_FALSE; + + const size_t vgSize = (size_t)locn.Size; + if (vgSize == 0) + return S_FALSE; + + vgBuf.Alloc(vgSize); + + _cfgPos = offset + locn.Offset; + _cfgSize = vgSize; + RINOK(InStream_SeekSet(stream, _cfgPos)) + RINOK(ReadStream_FALSE(stream, vgBuf, vgSize)) + if (locn.Checksum != LvmCrcCalc(vgBuf, vgSize)) + return S_FALSE; + + { + AString s; + s.SetFrom_CalcLen((const char *)(const Byte *)vgBuf, (unsigned)vgSize); + _cfgSize = s.Len(); + if (!cfg.Parse(s)) + return S_FALSE; + // isFinded = true; + // break; + } + } + + // if (isFinded) break; + } + + // if (!isFinded) return S_FALSE; + + if (cfg.Root.Items.Size() != 1) + return S_FALSE; + const CConfigItem &volGroup = cfg.Root.Items[0]; + if (volGroup.Name != "VolGroup00") + return S_FALSE; + + volGroup.GetPropVal_String("id", _id); + + if (!cfg.Root.GetPropVal_Number("creation_time", _cTime)) + _cTime = 0; + + UInt64 extentSize; + if (!volGroup.GetPropVal_Number("extent_size", extentSize)) + return S_FALSE; + + _extentSizeBits = GetLog(extentSize); + if (_extentSizeBits < 0 || _extentSizeBits > (62 - 9)) + return S_FALSE; + + + { + int pvsIndex = volGroup.FindSubItem("physical_volumes"); + if (pvsIndex < 0) + return S_FALSE; + + const CConfigItem &phyVols = volGroup.Items[pvsIndex]; + + FOR_VECTOR (i, phyVols.Items) + { + const CConfigItem &ci = phyVols.Items[i]; + CPhyVol pv; + if (!pv.Parse(ci)) + return S_FALSE; + _phyVols.Add(pv); + } + } + + { + int lvIndex = volGroup.FindSubItem("logical_volumes"); + if (lvIndex < 0) + return S_FALSE; + + const CConfigItem &logVolumes = volGroup.Items[lvIndex]; + + FOR_VECTOR (i, logVolumes.Items) + { + const CConfigItem &ci = logVolumes.Items[i]; + CLogVol &lv = _logVols.AddNew(); + lv.Parse(ci) ; // check error + } + } + + { + FOR_VECTOR (i, _logVols) + { + CLogVol &lv = _logVols[i]; + + CItem item; + + item.LogVol = (int)i; + item.Pos = 0; + item.Size = 0; + item.Name = lv.Name; + + if (lv.IsSupported) + { + UInt64 numExtents; + lv.IsSupported = lv.GetNumExtents(numExtents); + + if (lv.IsSupported) + { + lv.IsSupported = false; + item.Size = numExtents << (_extentSizeBits + 9); + + if (lv.Segments.Size() == 1) + { + const CSegment &segment = lv.Segments[0]; + if (segment.stripes.Size() == 1) + { + const CStripe &stripe = segment.stripes[0]; + FOR_VECTOR (pvIndex, _phyVols) + { + const CPhyVol &pv = _phyVols[pvIndex]; + if (pv.Name == stripe.Name) + { + item.Pos = (pv.pe_start + (stripe.ExtentOffset << _extentSizeBits)) << 9; + lv.IsSupported = true; + item.IsSupported = true; + break; + } + } + } + } + } + } + + _items.Add(item); + } + } + + { + FOR_VECTOR (i, _phyVols) + { + const CPhyVol &pv = _phyVols[i]; + + if (pv.pe_start > (UInt64)1 << (62 - 9)) + return S_FALSE; + if (pv.pe_count > (UInt64)1 << (62 - 9 - _extentSizeBits)) + return S_FALSE; + + CItem item; + + item.PhyVol = (int)i; + item.Pos = pv.pe_start << 9; + item.Size = pv.pe_count << (_extentSizeBits + 9); + item.Name = pv.Name; + item.IsSupported = true; + + _items.Add(item); + } + } + + return S_OK; +} + + +Z7_COM7F_IMF(CHandler::Open(IInStream *stream, + const UInt64 * /* maxCheckStartPosition */, + IArchiveOpenCallback * /* openArchiveCallback */)) +{ + COM_TRY_BEGIN + Close(); + RINOK(Open2(stream)) + _stream = stream; + return S_OK; + COM_TRY_END +} + + +Z7_COM7F_IMF(CHandler::Close()) +{ + CVolGroup::Clear(); + + _cfgPos = 0; + _cfgSize = 0; + _cTime = 0; + _phySize = 0; + _isArc = false; + _items.Clear(); + + _stream.Release(); + return S_OK; +} + + +static const Byte kProps[] = +{ + kpidPath, + kpidSize, + kpidFileSystem, + kpidCharacts, + kpidOffset, + kpidId +}; + +static const Byte kArcProps[] = +{ + kpidId, + kpidCTime, + kpidClusterSize +}; + +IMP_IInArchive_Props +IMP_IInArchive_ArcProps + +Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) +{ + COM_TRY_BEGIN + NCOM::CPropVariant prop; + + switch (propID) + { + case kpidMainSubfile: + { + /* + if (_items.Size() == 1) + prop = (UInt32)0; + */ + break; + } + case kpidPhySize: if (_phySize != 0) prop = _phySize; break; + case kpidId: + { + prop = _id; + break; + } + case kpidClusterSize: + { + if (_extentSizeBits >= 0) + prop = ((UInt64)1 << (_extentSizeBits + 9)); + break; + } + case kpidCTime: + { + if (_cTime != 0) + { + FILETIME ft; + NTime::UnixTime64_To_FileTime((Int64)_cTime, ft); + prop = ft; + } + break; + } + case kpidErrorFlags: + { + UInt32 v = 0; + if (!_isArc) v |= kpv_ErrorFlags_IsNotArc; + // if (_unsupported) v |= kpv_ErrorFlags_UnsupportedMethod; + // if (_headerError) v |= kpv_ErrorFlags_HeadersError; + if (v == 0 && !_stream) + v |= kpv_ErrorFlags_UnsupportedMethod; + if (v != 0) + prop = v; + break; + } + } + + prop.Detach(value); + return S_OK; + COM_TRY_END +} + + +Z7_COM7F_IMF(CHandler::GetNumberOfItems(UInt32 *numItems)) +{ + *numItems = _items.Size() + (_cfgSize == 0 ? 0 : 1); + return S_OK; +} + + +Z7_COM7F_IMF(CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *value)) +{ + COM_TRY_BEGIN + NCOM::CPropVariant prop; + + const CItem &item = _items[index]; + + // const CLogVol &item = _items[index]; + + if (index >= _items.Size()) + { + switch (propID) + { + case kpidPath: + { + prop = "meta.txt"; + break; + } + + case kpidSize: + case kpidPackSize: prop = _cfgSize; break; + } + } + else + { + switch (propID) + { + case kpidPath: + { + AString s = item.Name; + s += ".img"; + prop = s; + break; + } + + case kpidSize: + case kpidPackSize: prop = item.Size; break; + case kpidOffset: prop = item.Pos; break; + + case kpidId: + { + // prop = item.id; + break; + } + + // case kpidCharacts: FLAGS64_TO_PROP(g_PartitionFlags, item.Flags, prop); break; + } + } + + prop.Detach(value); + return S_OK; + COM_TRY_END +} + +REGISTER_ARC_I( + "LVM", "lvm", NULL, 0xBF, + k_Signature, + kSectorSize, + 0, + NULL) + +}} diff --git a/CPP/7zip/Archive/LzhHandler.cpp b/CPP/7zip/Archive/LzhHandler.cpp index 9239afd..adfe59d 100644 --- a/CPP/7zip/Archive/LzhHandler.cpp +++ b/CPP/7zip/Archive/LzhHandler.cpp @@ -4,6 +4,7 @@ #include "../../../C/CpuArch.h" +#include "../../Common/AutoPtr.h" #include "../../Common/ComTry.h" #include "../../Common/MyBuffer.h" #include "../../Common/StringConvert.h" @@ -37,6 +38,7 @@ using namespace NTime; static const UInt16 kCrc16Poly = 0xA001; +MY_ALIGN(64) static UInt16 g_LzhCrc16Table[256]; #define CRC16_UPDATE_BYTE(crc, b) (g_LzhCrc16Table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8)) @@ -219,7 +221,7 @@ struct CItem AString s (GetDirName()); const char kDirSeparator = '\\'; // check kDirSeparator in Linux - s.Replace((char)(unsigned char)0xFF, kDirSeparator); + s.Replace((char)(Byte)0xFF, kDirSeparator); if (!s.IsEmpty() && s.Back() != kDirSeparator) s += kDirSeparator; s += GetFileName(); @@ -552,8 +554,8 @@ Z7_COM7F_IMF(CHandler::Open(IInStream *stream, } if (_items.Size() % 100 == 0) { - UInt64 numFiles = _items.Size(); - UInt64 numBytes = item.DataPosition; + const UInt64 numFiles = _items.Size(); + const UInt64 numBytes = item.DataPosition; RINOK(callback->SetCompleted(&numFiles, &numBytes)) } } @@ -600,91 +602,76 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, } RINOK(extractCallback->SetTotal(totalUnPacked)) - UInt64 currentItemUnPacked, currentItemPacked; - - NCompress::NLzh::NDecoder::CCoder *lzhDecoderSpec = NULL; - CMyComPtr lzhDecoder; - // CMyComPtr lzh1Decoder; - // CMyComPtr arj2Decoder; - - NCompress::CCopyCoder *copyCoderSpec = new NCompress::CCopyCoder(); - CMyComPtr copyCoder = copyCoderSpec; + UInt32 cur_Unpacked, cur_Packed; - CLocalProgress *lps = new CLocalProgress; - CMyComPtr progress = lps; + CMyComPtr2_Create lps; lps->Init(extractCallback, false); - - CLimitedSequentialInStream *streamSpec = new CLimitedSequentialInStream; - CMyComPtr inStream(streamSpec); - streamSpec->SetStream(_stream); + CMyUniquePtr lzhDecoder; + CMyComPtr2_Create copyCoder; + CMyComPtr2_Create inStream; + inStream->SetStream(_stream); for (i = 0;; i++, - lps->OutSize += currentItemUnPacked, - lps->InSize += currentItemPacked) + lps->OutSize += cur_Unpacked, + lps->InSize += cur_Packed) { - currentItemUnPacked = 0; - currentItemPacked = 0; - + cur_Unpacked = 0; + cur_Packed = 0; RINOK(lps->SetCur()) - if (i >= numItems) break; - CMyComPtr realOutStream; - const Int32 askMode = testMode ? - NExtract::NAskMode::kTest : - NExtract::NAskMode::kExtract; - const UInt32 index = allFilesMode ? i : indices[i]; - const CItemEx &item = _items[index]; - RINOK(extractCallback->GetStream(index, &realOutStream, askMode)) - - if (item.IsDir()) + Int32 opRes; { - // if (!testMode) + CMyComPtr realOutStream; + const Int32 askMode = testMode ? + NExtract::NAskMode::kTest : + NExtract::NAskMode::kExtract; + const UInt32 index = allFilesMode ? i : indices[i]; + const CItemEx &item = _items[index]; + RINOK(extractCallback->GetStream(index, &realOutStream, askMode)) + + if (item.IsDir()) { - RINOK(extractCallback->PrepareOperation(askMode)) - RINOK(extractCallback->SetOperationResult(NExtract::NOperationResult::kOK)) + // if (!testMode) + { + RINOK(extractCallback->PrepareOperation(askMode)) + RINOK(extractCallback->SetOperationResult(NExtract::NOperationResult::kOK)) + } + continue; } - continue; - } - - if (!testMode && !realOutStream) - continue; - - RINOK(extractCallback->PrepareOperation(askMode)) - currentItemUnPacked = item.Size; - currentItemPacked = item.PackSize; + + if (!testMode && !realOutStream) + continue; + + RINOK(extractCallback->PrepareOperation(askMode)) + cur_Unpacked = item.Size; + cur_Packed = item.PackSize; - { - COutStreamWithCRC *outStreamSpec = new COutStreamWithCRC; - CMyComPtr outStream(outStreamSpec); - outStreamSpec->Init(realOutStream); + CMyComPtr2_Create outStream; + outStream->Init(realOutStream); realOutStream.Release(); RINOK(InStream_SeekSet(_stream, item.DataPosition)) - streamSpec->Init(item.PackSize); + inStream->Init(item.PackSize); HRESULT res = S_OK; - Int32 opRes = NExtract::NOperationResult::kOK; + opRes = NExtract::NOperationResult::kOK; if (item.IsCopyMethod()) { - res = copyCoder->Code(inStream, outStream, NULL, NULL, progress); - if (res == S_OK && copyCoderSpec->TotalSize != item.PackSize) + res = copyCoder.Interface()->Code(inStream, outStream, NULL, NULL, lps); + if (res == S_OK && copyCoder->TotalSize != item.PackSize) res = S_FALSE; } else if (item.IsLh4GroupMethod()) { - if (!lzhDecoder) - { - lzhDecoderSpec = new NCompress::NLzh::NDecoder::CCoder; - lzhDecoder = lzhDecoderSpec; - } - lzhDecoderSpec->FinishMode = true; - lzhDecoderSpec->SetDictSize(1 << item.GetNumDictBits()); - res = lzhDecoder->Code(inStream, outStream, NULL, ¤tItemUnPacked, progress); - if (res == S_OK && lzhDecoderSpec->GetInputProcessedSize() != item.PackSize) + lzhDecoder.Create_if_Empty(); + // lzhDecoder->FinishMode = true; + lzhDecoder->SetDictSize((UInt32)1 << item.GetNumDictBits()); + res = lzhDecoder->Code(inStream, outStream, cur_Unpacked, lps); + if (res == S_OK && lzhDecoder->GetInputProcessedSize() != item.PackSize) res = S_FALSE; } /* @@ -696,7 +683,7 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, lzh1Decoder = lzh1DecoderSpec; } lzh1DecoderSpec->SetDictionary(item.GetNumDictBits()); - res = lzh1Decoder->Code(inStream, outStream, NULL, ¤tItemUnPacked, progress); + res = lzh1Decoder->Code(inStream, outStream, NULL, &cur_Unpacked, progress); } */ else @@ -709,13 +696,12 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, else { RINOK(res) - if (outStreamSpec->GetCRC() != item.CRC) + if (outStream->GetCRC() != item.CRC) opRes = NExtract::NOperationResult::kCRCError; } } - outStream.Release(); - RINOK(extractCallback->SetOperationResult(opRes)) } + RINOK(extractCallback->SetOperationResult(opRes)) } return S_OK; COM_TRY_END diff --git a/CPP/7zip/Archive/LzmaHandler.cpp b/CPP/7zip/Archive/LzmaHandler.cpp index 11cb76e..7cbabf4 100644 --- a/CPP/7zip/Archive/LzmaHandler.cpp +++ b/CPP/7zip/Archive/LzmaHandler.cpp @@ -80,31 +80,26 @@ class CDecoder Z7_final { CMyComPtr _bcjStream; CFilterCoder *_filterCoder; - CMyComPtr _lzmaDecoder; public: - NCompress::NLzma::CDecoder *_lzmaDecoderSpec; + CMyComPtr2 _lzmaDecoder; ~CDecoder(); HRESULT Create(bool filtered, ISequentialInStream *inStream); HRESULT Code(const CHeader &header, ISequentialOutStream *outStream, ICompressProgressInfo *progress); - UInt64 GetInputProcessedSize() const { return _lzmaDecoderSpec->GetInputProcessedSize(); } + UInt64 GetInputProcessedSize() const { return _lzmaDecoder->GetInputProcessedSize(); } - void ReleaseInStream() { if (_lzmaDecoder) _lzmaDecoderSpec->ReleaseInStream(); } + void ReleaseInStream() { if (_lzmaDecoder) _lzmaDecoder->ReleaseInStream(); } HRESULT ReadInput(Byte *data, UInt32 size, UInt32 *processedSize) - { return _lzmaDecoderSpec->ReadFromInputStream(data, size, processedSize); } + { return _lzmaDecoder->ReadFromInputStream(data, size, processedSize); } }; HRESULT CDecoder::Create(bool filteredMode, ISequentialInStream *inStream) { - if (!_lzmaDecoder) - { - _lzmaDecoderSpec = new NCompress::NLzma::CDecoder; - _lzmaDecoderSpec->FinishStream = true; - _lzmaDecoder = _lzmaDecoderSpec; - } + _lzmaDecoder.Create_if_Empty(); + _lzmaDecoder->FinishStream = true; if (filteredMode) { @@ -117,7 +112,7 @@ HRESULT CDecoder::Create(bool filteredMode, ISequentialInStream *inStream) } } - return _lzmaDecoderSpec->SetInStream(inStream); + return _lzmaDecoder->SetInStream(inStream); } CDecoder::~CDecoder() @@ -131,7 +126,7 @@ HRESULT CDecoder::Code(const CHeader &header, ISequentialOutStream *outStream, if (header.FilterID > 1) return E_NOTIMPL; - RINOK(_lzmaDecoderSpec->SetDecoderProperties2(header.LzmaProps, 5)) + RINOK(_lzmaDecoder->SetDecoderProperties2(header.LzmaProps, 5)) bool filteredMode = (header.FilterID == 1); @@ -143,7 +138,7 @@ HRESULT CDecoder::Code(const CHeader &header, ISequentialOutStream *outStream, } const UInt64 *Size = header.HasSize() ? &header.Size : NULL; - HRESULT res = _lzmaDecoderSpec->CodeResume(outStream, Size, progress); + HRESULT res = _lzmaDecoder->CodeResume(outStream, Size, progress); if (filteredMode) { @@ -160,7 +155,7 @@ HRESULT CDecoder::Code(const CHeader &header, ISequentialOutStream *outStream, RINOK(res) if (header.HasSize()) - if (_lzmaDecoderSpec->GetOutputProcessedSize() != header.Size) + if (_lzmaDecoder->GetOutputProcessedSize() != header.Size) return S_FALSE; return S_OK; @@ -170,11 +165,7 @@ HRESULT CDecoder::Code(const CHeader &header, ISequentialOutStream *outStream, Z7_CLASS_IMP_CHandler_IInArchive_1( IArchiveOpenSeq ) - CHeader _header; bool _lzma86; - CMyComPtr _stream; - CMyComPtr _seqStream; - bool _isArc; bool _needSeekToStart; bool _dataAfterEnd; @@ -186,6 +177,10 @@ Z7_CLASS_IMP_CHandler_IInArchive_1( bool _unpackSize_Defined; bool _numStreams_Defined; + CHeader _header; + CMyComPtr _stream; + CMyComPtr _seqStream; + UInt64 _packSize; UInt64 _unpackSize; UInt64 _numStreams; @@ -220,6 +215,7 @@ Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) prop = v; break; } + default: break; } prop.Detach(value); return S_OK; @@ -288,6 +284,7 @@ Z7_COM7F_IMF(CHandler::GetProperty(UInt32 /* index */, PROPID propID, PROPVARIAN case kpidSize: if (_stream && _header.HasSize()) prop = _header.Size; break; case kpidPackSize: if (_packSize_Defined) prop = _packSize; break; case kpidMethod: GetMethod(prop); break; + default: break; } prop.Detach(value); return S_OK; @@ -366,7 +363,6 @@ Z7_COM7F_IMF(CHandler::Open(IInStream *inStream, const UInt64 *, IArchiveOpenCal ) return S_FALSE; - CDecoder state; const UInt32 outLimit = 1 << 11; Byte outBuf[outLimit]; @@ -452,9 +448,10 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, return E_INVALIDARG; if (_packSize_Defined) - extractCallback->SetTotal(_packSize); + RINOK(extractCallback->SetTotal(_packSize)) - + Int32 opResult; + { CMyComPtr realOutStream; const Int32 askMode = testMode ? NExtract::NAskMode::kTest : @@ -463,16 +460,14 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, if (!testMode && !realOutStream) return S_OK; - extractCallback->PrepareOperation(askMode); + RINOK(extractCallback->PrepareOperation(askMode)) - CDummyOutStream *outStreamSpec = new CDummyOutStream; - CMyComPtr outStream(outStreamSpec); - outStreamSpec->SetStream(realOutStream); - outStreamSpec->Init(); + CMyComPtr2_Create outStream; + outStream->SetStream(realOutStream); + outStream->Init(); realOutStream.Release(); - CLocalProgress *lps = new CLocalProgress; - CMyComPtr progress = lps; + CMyComPtr2_Create lps; lps->Init(extractCallback, true); if (_needSeekToStart) @@ -485,8 +480,7 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, _needSeekToStart = true; CDecoder decoder; - HRESULT result = decoder.Create(_lzma86, _seqStream); - RINOK(result) + RINOK(decoder.Create(_lzma86, _seqStream)) bool firstItem = true; @@ -496,6 +490,8 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, bool dataAfterEnd = false; + HRESULT hres = S_OK; + for (;;) { lps->InSize = packSize; @@ -523,32 +519,32 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, numStreams++; firstItem = false; - result = decoder.Code(st, outStream, progress); + hres = decoder.Code(st, outStream, lps); packSize = decoder.GetInputProcessedSize(); - unpackSize = outStreamSpec->GetSize(); + unpackSize = outStream->GetSize(); - if (result == E_NOTIMPL) + if (hres == E_NOTIMPL) { _unsupported = true; - result = S_FALSE; + hres = S_FALSE; break; } - if (result == S_FALSE) + if (hres == S_FALSE) break; - RINOK(result) + RINOK(hres) } if (firstItem) { _isArc = false; - result = S_FALSE; + hres = S_FALSE; } - else if (result == S_OK || result == S_FALSE) + else if (hres == S_OK || hres == S_FALSE) { if (dataAfterEnd) _dataAfterEnd = true; - else if (decoder._lzmaDecoderSpec->NeedsMoreInput()) + else if (decoder._lzmaDecoder->NeedsMoreInput()) _needMoreInput = true; _packSize = packSize; @@ -560,7 +556,7 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, _numStreams_Defined = true; } - Int32 opResult = NExtract::NOperationResult::kOK; + opResult = NExtract::NOperationResult::kOK; if (!_isArc) opResult = NExtract::NOperationResult::kIsNotArc; @@ -570,14 +566,15 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, opResult = NExtract::NOperationResult::kUnsupportedMethod; else if (_dataAfterEnd) opResult = NExtract::NOperationResult::kDataAfterEnd; - else if (result == S_FALSE) + else if (hres == S_FALSE) opResult = NExtract::NOperationResult::kDataError; - else if (result == S_OK) + else if (hres == S_OK) opResult = NExtract::NOperationResult::kOK; else - return result; + return hres; - outStream.Release(); + // outStream.Release(); + } return extractCallback->SetOperationResult(opResult); COM_TRY_END diff --git a/CPP/7zip/Archive/MbrHandler.cpp b/CPP/7zip/Archive/MbrHandler.cpp index 0d3611b..82d34fd 100644 --- a/CPP/7zip/Archive/MbrHandler.cpp +++ b/CPP/7zip/Archive/MbrHandler.cpp @@ -30,6 +30,11 @@ using namespace NWindows; namespace NArchive { + +namespace NFat { +API_FUNC_IsArc IsArc_Fat(const Byte *p, size_t size); +} + namespace NMbr { struct CChs @@ -89,8 +94,8 @@ struct CPartition bool IsExtended() const { return Type == 5 || Type == 0xF; } UInt32 GetLimit() const { return Lba + NumBlocks; } // bool IsActive() const { return Status == 0x80; } - UInt64 GetPos() const { return (UInt64)Lba * 512; } - UInt64 GetSize() const { return (UInt64)NumBlocks * 512; } + UInt64 GetPos(unsigned sectorSizeLog) const { return (UInt64)Lba << sectorSizeLog; } + UInt64 GetSize(unsigned sectorSizeLog) const { return (UInt64)NumBlocks << sectorSizeLog; } bool CheckLbaLimits() const { return (UInt32)0xFFFFFFFF - Lba >= NumBlocks; } bool Parse(const Byte *p) @@ -202,10 +207,13 @@ Z7_class_CHandler_final: public CHandlerCont UInt64 _totalSize; CByteBuffer _buffer; + UInt32 _signature; + unsigned _sectorSizeLog; + virtual int GetItem_ExtractInfo(UInt32 index, UInt64 &pos, UInt64 &size) const Z7_override Z7_final { const CItem &item = _items[index]; - pos = item.Part.GetPos(); + pos = item.Part.GetPos(_sectorSizeLog); size = item.Size; return NExtract::NOperationResult::kOK; } @@ -213,6 +221,18 @@ Z7_class_CHandler_final: public CHandlerCont HRESULT ReadTables(IInStream *stream, UInt32 baseLba, UInt32 lba, unsigned level); }; +/* +static bool IsEmptyBuffer(const Byte *data, size_t size) +{ + for (unsigned i = 0; i < size; i++) + if (data[i] != 0) + return false; + return true; +} +*/ + +const char *GetFileSystem(ISequentialInStream *stream, UInt64 partitionSize); + HRESULT CHandler::ReadTables(IInStream *stream, UInt32 baseLba, UInt32 lba, unsigned level) { if (level >= 128 || _items.Size() >= 128) @@ -221,24 +241,65 @@ HRESULT CHandler::ReadTables(IInStream *stream, UInt32 baseLba, UInt32 lba, unsi const unsigned kNumHeaderParts = 4; CPartition parts[kNumHeaderParts]; + if (level == 0) + _sectorSizeLog = 9; + const UInt32 kSectorSize = (UInt32)1 << _sectorSizeLog; + UInt32 bufSize = kSectorSize; + if (level == 0 && _totalSize >= (1 << 12)) + bufSize = (1 << 12); + _buffer.Alloc(bufSize); { - const UInt32 kSectorSize = 512; - _buffer.Alloc(kSectorSize); Byte *buf = _buffer; - UInt64 newPos = (UInt64)lba << 9; - if (newPos + 512 > _totalSize) + const UInt64 newPos = (UInt64)lba << _sectorSizeLog; + if (newPos + bufSize > _totalSize) return S_FALSE; RINOK(InStream_SeekSet(stream, newPos)) - RINOK(ReadStream_FALSE(stream, buf, kSectorSize)) - + RINOK(ReadStream_FALSE(stream, buf, bufSize)) if (buf[0x1FE] != 0x55 || buf[0x1FF] != 0xAA) return S_FALSE; - + if (level == 0) + _signature = GetUi32(buf + 0x1B8); for (unsigned i = 0; i < kNumHeaderParts; i++) if (!parts[i].Parse(buf + 0x1BE + 16 * i)) return S_FALSE; } + // 23.02: now we try to detect 4kn format (4 KB sectors case) + if (level == 0) + // if (_totalSize >= (1 << 28)) // we don't expect small images with 4 KB sectors + if (bufSize >= (1 << 12)) + { + UInt32 lastLim = 0; + UInt32 firstLba = 0; + UInt32 numBlocks = 0; // in first partition + for (unsigned i = 0; i < kNumHeaderParts; i++) + { + const CPartition &part = parts[i]; + if (part.IsEmpty()) + continue; + if (firstLba == 0 && part.NumBlocks != 0) + { + firstLba = part.Lba; + numBlocks = part.NumBlocks; + } + const UInt32 newLim = part.GetLimit(); + if (newLim < lastLim) + return S_FALSE; + lastLim = newLim; + } + if (lastLim != 0) + { + const UInt64 lim12 = (UInt64)lastLim << 12; + if (lim12 <= _totalSize + // && _totalSize - lim12 < (1 << 28) // we can try to exclude false positive cases + ) + // if (IsEmptyBuffer(&_buffer[(1 << 9)], (1 << 12) - (1 << 9))) + if (InStream_SeekSet(stream, (UInt64)firstLba << 12) == S_OK) + if (GetFileSystem(stream, (UInt64)numBlocks << 12)) + _sectorSizeLog = 12; + } + } + PRF(printf("\n# %8X", lba)); UInt32 limLba = lba + 1; @@ -265,7 +326,7 @@ HRESULT CHandler::ReadTables(IInStream *stream, UInt32 baseLba, UInt32 lba, unsi newLba = baseLba + part.Lba; if (newLba < limLba) return S_FALSE; - HRESULT res = ReadTables(stream, level < 1 ? newLba : baseLba, newLba, level + 1); + const HRESULT res = ReadTables(stream, level < 1 ? newLba : baseLba, newLba, level + 1); if (res != S_FALSE && res != S_OK) return res; } @@ -287,8 +348,8 @@ HRESULT CHandler::ReadTables(IInStream *stream, UInt32 baseLba, UInt32 lba, unsi else { const CItem &back = _items.Back(); - UInt32 backLimit = back.Part.GetLimit(); - UInt32 partLimit = part.GetLimit(); + const UInt32 backLimit = back.Part.GetLimit(); + const UInt32 partLimit = part.GetLimit(); if (backLimit < partLimit) { n.IsReal = false; @@ -302,7 +363,7 @@ HRESULT CHandler::ReadTables(IInStream *stream, UInt32 baseLba, UInt32 lba, unsi if (n.Part.GetLimit() < limLba) return S_FALSE; limLba = n.Part.GetLimit(); - n.Size = n.Part.GetSize(); + n.Size = n.Part.GetSize(_sectorSizeLog); _items.Add(n); } } @@ -310,27 +371,30 @@ HRESULT CHandler::ReadTables(IInStream *stream, UInt32 baseLba, UInt32 lba, unsi } +static const Byte k_Ntfs_Signature[] = { 'N', 'T', 'F', 'S', ' ', ' ', ' ', ' ', 0 }; + static bool Is_Ntfs(const Byte *p) { if (p[0x1FE] != 0x55 || p[0x1FF] != 0xAA) return false; - // int codeOffset = 0; switch (p[0]) { case 0xE9: /* codeOffset = 3 + (Int16)Get16(p + 1); */ break; case 0xEB: if (p[2] != 0x90) return false; /* codeOffset = 2 + (int)(signed char)p[1]; */ break; default: return false; } - return memcmp(p + 3, "NTFS ", 8) == 0; + return memcmp(p + 3, k_Ntfs_Signature, Z7_ARRAY_SIZE(k_Ntfs_Signature)) == 0; } +static const Byte k_ExFat_Signature[] = { 'E', 'X', 'F', 'A', 'T', ' ', ' ', ' ' }; + static bool Is_ExFat(const Byte *p) { if (p[0x1FE] != 0x55 || p[0x1FF] != 0xAA) return false; if (p[0] != 0xEB || p[1] != 0x76 || p[2] != 0x90) return false; - return memcmp(p + 3, "EXFAT ", 8) == 0; + return memcmp(p + 3, k_ExFat_Signature, Z7_ARRAY_SIZE(k_ExFat_Signature)) == 0; } static bool AllAreZeros(const Byte *p, size_t size) @@ -350,8 +414,7 @@ static bool Is_Udf(const Byte *p) } -static const char *GetFileSystem( - ISequentialInStream *stream, UInt64 partitionSize) +const char *GetFileSystem(ISequentialInStream *stream, UInt64 partitionSize) { const size_t kHeaderSize = 1 << 9; if (partitionSize >= kHeaderSize) @@ -364,6 +427,8 @@ static const char *GetFileSystem( return "NTFS"; if (Is_ExFat(buf)) return "exFAT"; + if (NFat::IsArc_Fat(buf, kHeaderSize)) + return "FAT"; const size_t kHeaderSize2 = k_Udf_StartPos + (1 << 9); if (partitionSize >= kHeaderSize2) { @@ -392,15 +457,17 @@ Z7_COM7F_IMF(CHandler::Open(IInStream *stream, RINOK(ReadTables(stream, 0, 0, 0)) if (_items.IsEmpty()) return S_FALSE; - UInt32 lbaLimit = _items.Back().Part.GetLimit(); - UInt64 lim = (UInt64)lbaLimit << 9; - if (lim < _totalSize) { - CItem n; - n.Part.Lba = lbaLimit; - n.Size = _totalSize - lim; - n.IsReal = false; - _items.Add(n); + const UInt32 lbaLimit = _items.Back().Part.GetLimit(); + const UInt64 lim = (UInt64)lbaLimit << _sectorSizeLog; + if (lim < _totalSize) + { + CItem n; + n.Part.Lba = lbaLimit; + n.Size = _totalSize - lim; + n.IsReal = false; + _items.Add(n); + } } FOR_VECTOR (i, _items) @@ -408,7 +475,7 @@ Z7_COM7F_IMF(CHandler::Open(IInStream *stream, CItem &item = _items[i]; if (item.Part.Type != kType_Windows_NTFS) continue; - if (InStream_SeekSet(stream, item.Part.GetPos()) != S_OK) + if (InStream_SeekSet(stream, item.Part.GetPos(_sectorSizeLog)) != S_OK) continue; item.FileSystem = GetFileSystem(stream, item.Size); item.WasParsed = true; @@ -447,8 +514,15 @@ static const CStatProp kProps[] = { "End CHS", kpidEndChs, VT_BSTR} }; +static const Byte kArcProps[] = +{ + kpidSectorSize, + kpidId +}; + IMP_IInArchive_Props_WITH_NAME -IMP_IInArchive_ArcProps_NO_Table +IMP_IInArchive_ArcProps + Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) { @@ -473,6 +547,8 @@ Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) break; } case kpidPhySize: prop = _totalSize; break; + case kpidSectorSize: prop = (UInt32)((UInt32)1 << _sectorSizeLog); break; + case kpidId: prop = _signature; break; } prop.Detach(value); return S_OK; @@ -544,7 +620,7 @@ Z7_COM7F_IMF(CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val break; case kpidSize: case kpidPackSize: prop = item.Size; break; - case kpidOffset: prop = part.GetPos(); break; + case kpidOffset: prop = part.GetPos(_sectorSizeLog); break; case kpidPrimary: if (item.IsReal) prop = item.IsPrim; break; case kpidBegChs: if (item.IsReal) part.BeginChs.ToString(prop); break; case kpidEndChs: if (item.IsReal) part.EndChs.ToString(prop); break; diff --git a/CPP/7zip/Archive/MslzHandler.cpp b/CPP/7zip/Archive/MslzHandler.cpp index 04549f8..daf58bd 100644 --- a/CPP/7zip/Archive/MslzHandler.cpp +++ b/CPP/7zip/Archive/MslzHandler.cpp @@ -75,6 +75,7 @@ Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) if (_needMoreInput) v |= kpv_ErrorFlags_UnexpectedEnd; if (_dataAfterEnd) v |= kpv_ErrorFlags_DataAfterEnd; prop = v; + break; } } prop.Detach(value); @@ -152,7 +153,7 @@ void CHandler::ParseName(Byte replaceByte, IArchiveOpenCallback *callback) } if (replaceByte >= 0x20 && replaceByte < 0x80) - _name += (char)replaceByte; + _name.Add_Char((char)replaceByte); } Z7_COM7F_IMF(CHandler::Open(IInStream *stream, const UInt64 * /* maxCheckStartPosition */, @@ -292,8 +293,9 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, if (numItems != (UInt32)(Int32)-1 && (numItems != 1 || indices[0] != 0)) return E_INVALIDARG; - // extractCallback->SetTotal(_unpackSize); - + // RINOK(extractCallback->SetTotal(_unpackSize)) + Int32 opRes; + { CMyComPtr realOutStream; const Int32 askMode = testMode ? NExtract::NAskMode::kTest : @@ -302,16 +304,14 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, if (!testMode && !realOutStream) return S_OK; - extractCallback->PrepareOperation(askMode); + RINOK(extractCallback->PrepareOperation(askMode)) - CDummyOutStream *outStreamSpec = new CDummyOutStream; - CMyComPtr outStream(outStreamSpec); - outStreamSpec->SetStream(realOutStream); - outStreamSpec->Init(); - realOutStream.Release(); + CMyComPtr2_Create outStream; + outStream->SetStream(realOutStream); + outStream->Init(); + // realOutStream.Release(); - CLocalProgress *lps = new CLocalProgress; - CMyComPtr progress = lps; + CMyComPtr2_Create lps; lps->Init(extractCallback, false); if (_needSeekToStart) @@ -323,7 +323,7 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, else _needSeekToStart = true; - Int32 opRes = NExtract::NOperationResult::kDataError; + opRes = NExtract::NOperationResult::kDataError; bool isArc = false; bool needMoreInput = false; @@ -344,7 +344,7 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, unpackSize = GetUi32(buffer + 10); if (unpackSize <= kUnpackSizeMax) { - HRESULT result = MslzDec(s, outStream, unpackSize, needMoreInput, progress); + const HRESULT result = MslzDec(s, outStream, unpackSize, needMoreInput, lps); if (result == S_OK) opRes = NExtract::NOperationResult::kOK; else if (result != S_FALSE) @@ -374,8 +374,7 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, opRes = NExtract::NOperationResult::kUnexpectedEnd; else if (_dataAfterEnd) opRes = NExtract::NOperationResult::kDataAfterEnd; - - outStream.Release(); + } return extractCallback->SetOperationResult(opRes); COM_TRY_END } diff --git a/CPP/7zip/Archive/Nsis/NsisHandler.cpp b/CPP/7zip/Archive/Nsis/NsisHandler.cpp index 7a512b7..7ce2e8e 100644 --- a/CPP/7zip/Archive/Nsis/NsisHandler.cpp +++ b/CPP/7zip/Archive/Nsis/NsisHandler.cpp @@ -73,7 +73,7 @@ static void AddDictProp(AString &s, UInt32 val) if ((val & ((1 << 20) - 1)) == 0) { val >>= 20; c = 'm'; } else if ((val & ((1 << 10) - 1)) == 0) { val >>= 10; c = 'k'; } s.Add_UInt32(val); - s += c; + s.Add_Char(c); } static AString GetMethod(bool useFilter, NMethodType::EEnum method, UInt32 dict) @@ -87,7 +87,7 @@ static AString GetMethod(bool useFilter, NMethodType::EEnum method, UInt32 dict) s += ((unsigned)method < Z7_ARRAY_SIZE(kMethods)) ? kMethods[(unsigned)method] : kUnknownMethod; if (method == NMethodType::kLZMA) { - s += ':'; + s.Add_Colon(); AddDictProp(s, dict); } return s; @@ -105,7 +105,7 @@ AString CHandler::GetMethod(NMethodType::EEnum method, bool useItemFilter, UInt3 s += (method < Z7_ARRAY_SIZE(kMethods)) ? kMethods[method] : kUnknownMethod; if (method == NMethodType::kLZMA) { - s += ':'; + s.Add_Colon(); s += GetStringForSizeValue(_archive.IsSolid ? _archive.DictionarySize: dictionary); } return s; @@ -427,8 +427,7 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, extractCallback->SetTotal(totalSize + solidPosMax); - CLocalProgress *lps = new CLocalProgress; - CMyComPtr progress = lps; + CMyComPtr2_Create lps; lps->Init(extractCallback, !_archive.IsSolid); if (_archive.IsSolid) @@ -559,7 +558,7 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, } else { - HRESULT res = _archive.Decoder.SetToPos(pos, progress); + HRESULT res = _archive.Decoder.SetToPos(pos, lps); if (res != S_OK) { if (res != S_FALSE) @@ -620,7 +619,7 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, writeToTemp1 ? &tempBuf : NULL, is_PatchedUninstaller, item.PatchSize, is_PatchedUninstaller ? NULL : (ISequentialOutStream *)realOutStream, - progress, + lps, curPacked, curUnpacked32); curUnpacked = curUnpacked32; if (_archive.IsSolid) @@ -665,7 +664,7 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, writeToTemp ? &tempBuf2 : NULL, false, 0, realOutStream, - progress, + lps, curPacked2, curUnpacked2); curPacked += curPacked2; if (!_archive.IsSolid) diff --git a/CPP/7zip/Archive/Nsis/NsisIn.cpp b/CPP/7zip/Archive/Nsis/NsisIn.cpp index 05ebfd0..c9e2c01 100644 --- a/CPP/7zip/Archive/Nsis/NsisIn.cpp +++ b/CPP/7zip/Archive/Nsis/NsisIn.cpp @@ -979,7 +979,7 @@ void CInArchive::GetNsisString_Unicode_Raw(const Byte *p) break; if (c < 0x80) { - Raw_UString += (char)c; + Raw_UString.Add_Char((char)c); continue; } @@ -2623,7 +2623,7 @@ void CInArchive::DetectNsisType(const CBlockHeader &bh, const Byte *p) for (UInt32 kkk = 0; kkk < bh.Num; kkk++, p2 += kCmdSize) { - UInt32 cmd = Get32(p2); // we use original (not converted) command + const UInt32 cmd = Get32(p2); // we use original (not converted) command if (cmd < EW_WRITEUNINSTALLER || cmd > EW_WRITEUNINSTALLER + numInsertMax) @@ -2639,7 +2639,7 @@ void CInArchive::DetectNsisType(const CBlockHeader &bh, const Byte *p) params[3] <= 1) continue; - UInt32 altParam = params[3]; + const UInt32 altParam = params[3]; if (!IsGoodString(params[0]) || !IsGoodString(altParam)) continue; @@ -2649,8 +2649,8 @@ void CInArchive::DetectNsisType(const CBlockHeader &bh, const Byte *p) continue; if (AreTwoParamStringsEqual(altParam + additional, params[0])) { - unsigned numInserts = cmd - EW_WRITEUNINSTALLER; - mask |= (1 << numInserts); + const unsigned numInserts = cmd - EW_WRITEUNINSTALLER; + mask |= ((unsigned)1 << numInserts); } } diff --git a/CPP/7zip/Archive/Nsis/NsisIn.h b/CPP/7zip/Archive/Nsis/NsisIn.h index bb8de62..1f8c967 100644 --- a/CPP/7zip/Archive/Nsis/NsisIn.h +++ b/CPP/7zip/Archive/Nsis/NsisIn.h @@ -416,7 +416,7 @@ class CInArchive s = MultiByteToUnicodeString(APrefixes[item.Prefix]); if (s.Len() > 0) if (s.Back() != L'\\') - s += '\\'; + s.Add_Char('\\'); } if (IsUnicode) diff --git a/CPP/7zip/Archive/NtfsHandler.cpp b/CPP/7zip/Archive/NtfsHandler.cpp index c13e0ca..d55521d 100644 --- a/CPP/7zip/Archive/NtfsHandler.cpp +++ b/CPP/7zip/Archive/NtfsHandler.cpp @@ -71,14 +71,15 @@ struct CHeader { unsigned SectorSizeLog; unsigned ClusterSizeLog; + unsigned MftRecordSizeLog; // Byte MediaType; - UInt32 NumHiddenSectors; + // UInt32 NumHiddenSectors; UInt64 NumSectors; UInt64 NumClusters; UInt64 MftCluster; UInt64 SerialNumber; - UInt16 SectorsPerTrack; - UInt16 NumHeads; + // UInt16 SectorsPerTrack; + // UInt16 NumHeads; UInt64 GetPhySize_Clusters() const { return NumClusters << ClusterSizeLog; } UInt64 GetPhySize_Max() const { return (NumSectors + 1) << SectorSizeLog; } @@ -111,30 +112,42 @@ bool CHeader::Parse(const Byte *p) if (memcmp(p + 3, "NTFS ", 8) != 0) return false; { - int t = GetLog(Get16(p + 11)); - if (t < 9 || t > 12) - return false; - SectorSizeLog = (unsigned)t; - t = GetLog(p[13]); - if (t < 0) - return false; - sectorsPerClusterLog = (unsigned)t; - ClusterSizeLog = SectorSizeLog + sectorsPerClusterLog; - if (ClusterSizeLog > 30) - return false; + { + const int t = GetLog(Get16(p + 11)); + if (t < 9 || t > 12) + return false; + SectorSizeLog = (unsigned)t; + } + { + const unsigned v = p[13]; + if (v <= 0x80) + { + const int t = GetLog(v); + if (t < 0) + return false; + sectorsPerClusterLog = (unsigned)t; + } + else + sectorsPerClusterLog = 0x100 - v; + ClusterSizeLog = SectorSizeLog + sectorsPerClusterLog; + if (ClusterSizeLog > 30) + return false; + } } for (int i = 14; i < 21; i++) if (p[i] != 0) return false; + // F8 : a hard disk + // F0 : high-density 3.5-inch floppy disk if (p[21] != 0xF8) // MediaType = Fixed_Disk return false; if (Get16(p + 22) != 0) // NumFatSectors return false; - G16(p + 24, SectorsPerTrack); // 63 usually - G16(p + 26, NumHeads); // 255 - G32(p + 28, NumHiddenSectors); // 63 (XP) / 2048 (Vista and win7) / (0 on media that are not partitioned ?) + // G16(p + 24, SectorsPerTrack); // 63 usually + // G16(p + 26, NumHeads); // 255 + // G32(p + 28, NumHiddenSectors); // 63 (XP) / 2048 (Vista and win7) / (0 on media that are not partitioned ?) if (Get32(p + 32) != 0) // NumSectors32 return false; @@ -156,14 +169,47 @@ bool CHeader::Parse(const Byte *p) NumClusters = NumSectors >> sectorsPerClusterLog; - G64(p + 0x30, MftCluster); + G64(p + 0x30, MftCluster); // $MFT. // G64(p + 0x38, Mft2Cluster); - G64(p + 0x48, SerialNumber); - UInt32 numClustersInMftRec; - UInt32 numClustersInIndexBlock; - G32(p + 0x40, numClustersInMftRec); // -10 means 2 ^10 = 1024 bytes. - G32(p + 0x44, numClustersInIndexBlock); - return (numClustersInMftRec < 256 && numClustersInIndexBlock < 256); + G64(p + 0x48, SerialNumber); // $MFTMirr + + /* + numClusters_per_MftRecord: + numClusters_per_IndexBlock: + only low byte from 4 bytes is used. Another 3 high bytes are zeros. + If the number is positive (number < 0x80), + then it represents the number of clusters. + If the number is negative (number >= 0x80), + then the size of the file record is 2 raised to the absolute value of this number. + example: (0xF6 == -10) means 2^10 = 1024 bytes. + */ + { + UInt32 numClusters_per_MftRecord; + G32(p + 0x40, numClusters_per_MftRecord); + if (numClusters_per_MftRecord >= 0x100 || numClusters_per_MftRecord == 0) + return false; + if (numClusters_per_MftRecord < 0x80) + { + const int t = GetLog(numClusters_per_MftRecord); + if (t < 0) + return false; + MftRecordSizeLog = (unsigned)t + ClusterSizeLog; + } + else + MftRecordSizeLog = 0x100 - numClusters_per_MftRecord; + // what exact MFT record sizes are possible and supported by Windows? + // do we need to change this limit here? + const unsigned k_MftRecordSizeLog_MAX = 12; + if (MftRecordSizeLog > k_MftRecordSizeLog_MAX) + return false; + if (MftRecordSizeLog < SectorSizeLog) + return false; + } + { + UInt32 numClusters_per_IndexBlock; + G32(p + 0x44, numClusters_per_IndexBlock); + return (numClusters_per_IndexBlock < 0x100); + } } struct CMftRef @@ -242,7 +288,7 @@ struct CFileNameAttr {} }; -static void GetString(const Byte *p, unsigned len, UString2 &res) +static void GetString(const Byte *p, const unsigned len, UString2 &res) { if (len == 0 && res.IsEmpty()) return; @@ -250,7 +296,7 @@ static void GetString(const Byte *p, unsigned len, UString2 &res) unsigned i; for (i = 0; i < len; i++) { - wchar_t c = Get16(p + i * 2); + const wchar_t c = Get16(p + i * 2); if (c == 0) break; s[i] = c; @@ -273,8 +319,8 @@ bool CFileNameAttr::Parse(const Byte *p, unsigned size) G32(p + 0x38, Attrib); // G16(p + 0x3C, PackedEaSize); NameType = p[0x41]; - unsigned len = p[0x40]; - if (0x42 + len > size) + const unsigned len = p[0x40]; + if (0x42 + len * 2 > size) return false; if (len != 0) GetString(p + 0x42, len, Name); @@ -537,11 +583,11 @@ bool CAttr::ParseExtents(CRecordVector &extents, UInt64 numClustersMax, while (size > 0) { - Byte b = *p++; + const unsigned b = *p++; size--; if (b == 0) break; - UInt32 num = b & 0xF; + unsigned num = b & 0xF; if (num == 0 || num > 8 || num > size) return false; @@ -561,7 +607,7 @@ bool CAttr::ParseExtents(CRecordVector &extents, UInt64 numClustersMax, e.Virt = vcn; vcn += vSize; - num = (b >> 4) & 0xF; + num = b >> 4; if (num > 8 || num > size) return false; @@ -614,12 +660,9 @@ static const UInt64 kEmptyTag = (UInt64)(Int64)-1; static const unsigned kNumCacheChunksLog = 1; static const size_t kNumCacheChunks = (size_t)1 << kNumCacheChunksLog; -Z7_CLASS_IMP_COM_1( +Z7_CLASS_IMP_IInStream( CInStream - , IInStream ) - Z7_IFACE_COM7_IMP(ISequentialInStream) - UInt64 _virtPos; UInt64 _physPos; UInt64 _curRem; @@ -1279,12 +1322,12 @@ bool CMftRec::Parse(Byte *p, unsigned sectorSizeLog, UInt32 numSectors, UInt32 r G16(p + 0x10, SeqNumber); // G16(p + 0x12, LinkCount); // PRF(printf(" L=%d", LinkCount)); - UInt32 attrOffs = Get16(p + 0x14); + const UInt32 attrOffs = Get16(p + 0x14); G16(p + 0x16, Flags); PRF(printf(" F=%4X", Flags)); - UInt32 bytesInUse = Get32(p + 0x18); - UInt32 bytesAlloc = Get32(p + 0x1C); + const UInt32 bytesInUse = Get32(p + 0x18); + const UInt32 bytesAlloc = Get32(p + 0x1C); G64(p + 0x20, BaseMftRef.Val); if (BaseMftRef.Val != 0) { @@ -1405,7 +1448,6 @@ struct CDatabase CObjectVector Recs; CMyComPtr InStream; CHeader Header; - unsigned RecSizeLog; UInt64 PhySize; IArchiveOpenCallback *OpenCallback; @@ -1417,6 +1459,9 @@ struct CDatabase CByteBuffer SecurData; CRecordVector SecurOffsets; + // bool _headerWarning; + bool ThereAreAltStreams; + bool _showSystemFiles; bool _showDeletedFiles; CObjectVector VirtFolderNames; @@ -1426,10 +1471,6 @@ struct CDatabase int _lostFolderIndex_Normal; int _lostFolderIndex_Deleted; - // bool _headerWarning; - - bool ThereAreAltStreams; - void InitProps() { _showSystemFiles = true; @@ -1449,7 +1490,7 @@ struct CDatabase HRESULT SeekToCluster(UInt64 cluster); - int FindDirItemForMtfRec(UInt64 recIndex) const + int Find_DirItem_For_MftRec(UInt64 recIndex) const { if (recIndex >= Recs.Size()) return -1; @@ -1773,26 +1814,22 @@ HRESULT CDatabase::Open() SeekToCluster(Header.MftCluster); - CMftRec mftRec; - UInt32 numSectorsInRec; - + // we use ByteBuf for records reading. + // so the size of ByteBuf must be >= mftRecordSize + const size_t recSize = (size_t)1 << Header.MftRecordSizeLog; + const size_t kBufSize = MyMax((size_t)(1 << 15), recSize); + ByteBuf.Alloc(kBufSize); + RINOK(ReadStream_FALSE(InStream, ByteBuf, recSize)) + { + const UInt32 allocSize = Get32(ByteBuf + 0x1C); + if (allocSize != recSize) + return S_FALSE; + } + // MftRecordSizeLog >= SectorSizeLog + const UInt32 numSectorsInRec = 1u << (Header.MftRecordSizeLog - Header.SectorSizeLog); CMyComPtr mftStream; + CMftRec mftRec; { - UInt32 blockSize = 1 << 12; - ByteBuf.Alloc(blockSize); - RINOK(ReadStream_FALSE(InStream, ByteBuf, blockSize)) - - { - const UInt32 allocSize = Get32(ByteBuf + 0x1C); - const int t = GetLog(allocSize); - if (t < (int)Header.SectorSizeLog) - return S_FALSE; - RecSizeLog = (unsigned)t; - if (RecSizeLog > 15) - return S_FALSE; - } - - numSectorsInRec = 1 << (RecSizeLog - Header.SectorSizeLog); if (!mftRec.Parse(ByteBuf, Header.SectorSizeLog, numSectorsInRec, 0, NULL)) return S_FALSE; if (!mftRec.Is_Magic_FILE()) @@ -1807,25 +1844,18 @@ HRESULT CDatabase::Open() // CObjectVector SecurityAttrs; - UInt64 mftSize = mftRec.DataAttrs[0].Size; + const UInt64 mftSize = mftRec.DataAttrs[0].Size; if ((mftSize >> 4) > Header.GetPhySize_Clusters()) return S_FALSE; - const size_t kBufSize = (1 << 15); - const size_t recSize = ((size_t)1 << RecSizeLog); - if (kBufSize < recSize) - return S_FALSE; - { - const UInt64 numFiles = mftSize >> RecSizeLog; + const UInt64 numFiles = mftSize >> Header.MftRecordSizeLog; if (numFiles > (1 << 30)) return S_FALSE; if (OpenCallback) { RINOK(OpenCallback->SetTotal(&numFiles, &mftSize)) } - - ByteBuf.Alloc(kBufSize); Recs.ClearAndReserve((unsigned)numFiles); } @@ -2119,7 +2149,7 @@ HRESULT CDatabase::Open() item.ParentFolder = -1; else { - int index = FindDirItemForMtfRec(refIndex); + int index = Find_DirItem_For_MftRec(refIndex); if (index < 0 || Recs[Items[index].RecIndex].SeqNumber != parentDirRef.GetNumber()) { @@ -2372,7 +2402,7 @@ static const CStatProp kArcProps[] = { NULL, kpidFileSystem, VT_BSTR}, { NULL, kpidClusterSize, VT_UI4}, { NULL, kpidSectorSize, VT_UI4}, - { "Record Size", kpidRecordSize, VT_UI4}, + { "MFT Record Size", kpidRecordSize, VT_UI4}, { NULL, kpidHeadersSize, VT_UI8}, { NULL, kpidCTime, VT_FILETIME}, { NULL, kpidId, VT_UI8}, @@ -2474,7 +2504,7 @@ Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) break; } case kpidSectorSize: prop = (UInt32)1 << Header.SectorSizeLog; break; - case kpidRecordSize: prop = (UInt32)1 << RecSizeLog; break; + case kpidRecordSize: prop = (UInt32)1 << Header.MftRecordSizeLog; break; case kpidId: prop = Header.SerialNumber; break; case kpidIsTree: prop = true; break; @@ -2745,21 +2775,19 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, UInt32 clusterSize = Header.ClusterSize(); CByteBuffer buf(clusterSize); - NCompress::CCopyCoder *copyCoderSpec = new NCompress::CCopyCoder(); - CMyComPtr copyCoder = copyCoderSpec; - - CLocalProgress *lps = new CLocalProgress; - CMyComPtr progress = lps; + CMyComPtr2_Create lps; lps->Init(extractCallback, false); + CMyComPtr2_Create copyCoder; + CMyComPtr2_Create outStream; - CDummyOutStream *outStreamSpec = new CDummyOutStream; - CMyComPtr outStream(outStreamSpec); - - for (i = 0; i < numItems; i++) + for (i = 0;; i++) { lps->InSize = totalPackSize; lps->OutSize = totalSize; RINOK(lps->SetCur()) + if (i >= numItems) + break; + CMyComPtr realOutStream; const Int32 askMode = testMode ? NExtract::NAskMode::kTest : @@ -2780,9 +2808,9 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, continue; RINOK(extractCallback->PrepareOperation(askMode)) - outStreamSpec->SetStream(realOutStream); + outStream->SetStream(realOutStream); realOutStream.Release(); - outStreamSpec->Init(); + outStream->Init(); const CMftRec &rec = Recs[item.RecIndex]; @@ -2797,7 +2825,7 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, RINOK(hres) if (inStream) { - hres = copyCoder->Code(inStream, outStream, NULL, NULL, progress); + hres = copyCoder.Interface()->Code(inStream, outStream, NULL, NULL, lps); if (hres != S_OK && hres != S_FALSE) { RINOK(hres) @@ -2813,7 +2841,7 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, totalPackSize += data.GetPackSize(); totalSize += data.GetSize(); } - outStreamSpec->ReleaseStream(); + outStream->ReleaseStream(); RINOK(extractCallback->SetOperationResult(res)) } return S_OK; @@ -2843,6 +2871,12 @@ Z7_COM7F_IMF(CHandler::SetProperties(const wchar_t * const *names, const PROPVAR { RINOK(PROPVARIANT_to_bool(prop, _showSystemFiles)) } + else if (IsString1PrefixedByString2_NoCase_Ascii(name, "mt")) + { + } + else if (IsString1PrefixedByString2_NoCase_Ascii(name, "memuse")) + { + } else return E_INVALIDARG; } diff --git a/CPP/7zip/Archive/PeHandler.cpp b/CPP/7zip/Archive/PeHandler.cpp index 9851ed3..0cab820 100644 --- a/CPP/7zip/Archive/PeHandler.cpp +++ b/CPP/7zip/Archive/PeHandler.cpp @@ -516,8 +516,15 @@ static const CUInt32PCharPair g_MachinePairs[] = { 0x0520, "TriCore" }, { 0x0CEF, "CEF" }, { 0x0EBC, "EFI" }, + { 0x5032, "RISCV32" }, + { 0x5064, "RISCV64" }, +// { 0x5128, "RISCV128" }, + { 0x6232, "LOONGARCH32" }, + { 0x6264, "LOONGARCH64" }, { 0x8664, "x64" }, { 0x9041, "M32R" }, + { 0xA641, "ARM64EC" }, + { 0xA64e, "ARM64X" }, { 0xAA64, "ARM64" }, { 0xC0EE, "CEE" } }; diff --git a/CPP/7zip/Archive/PpmdHandler.cpp b/CPP/7zip/Archive/PpmdHandler.cpp index e5c2f9b..5f9e1c3 100644 --- a/CPP/7zip/Archive/PpmdHandler.cpp +++ b/CPP/7zip/Archive/PpmdHandler.cpp @@ -74,7 +74,7 @@ HRESULT CItem::ReadHeader(ISequentialInStream *s, UInt32 &headerSize) return S_FALSE; Attrib = GetUi32(h + 4); Time = GetUi32(h + 12); - unsigned info = GetUi16(h + 8); + const unsigned info = GetUi16(h + 8); Order = (info & 0xF) + 1; MemInMB = ((info >> 4) & 0xFF) + 1; Ver = info >> 12; @@ -90,7 +90,7 @@ HRESULT CItem::ReadHeader(ISequentialInStream *s, UInt32 &headerSize) if (nameLen > (1 << 9)) return S_FALSE; char *name = Name.GetBuf(nameLen); - HRESULT res = ReadStream_FALSE(s, name, nameLen); + const HRESULT res = ReadStream_FALSE(s, name, nameLen); Name.ReleaseBuf_CalcLen(nameLen); headerSize = kHeaderSize + nameLen; return res; @@ -128,12 +128,12 @@ IMP_IInArchive_ArcProps void CHandler::GetVersion(NCOM::CPropVariant &prop) { AString s ("PPMd"); - s += (char)('A' + _item.Ver); + s.Add_Char((char)('A' + _item.Ver)); s += ":o"; s.Add_UInt32(_item.Order); s += ":mem"; s.Add_UInt32(_item.MemInMB); - s += 'm'; + s.Add_Char('m'); if (_item.Ver >= kNewHeaderVer && _item.Restor != 0) { s += ":r"; @@ -287,6 +287,8 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, // extractCallback->SetTotal(_packSize); UInt64 currentTotalPacked = 0; RINOK(extractCallback->SetCompleted(¤tTotalPacked)) + Int32 opRes; +{ CMyComPtr realOutStream; const Int32 askMode = testMode ? NExtract::NAskMode::kTest : @@ -295,7 +297,7 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, if (!testMode && !realOutStream) return S_OK; - extractCallback->PrepareOperation(askMode); + RINOK(extractCallback->PrepareOperation(askMode)) CByteInBufWrap inBuf; if (!inBuf.Alloc(1 << 20)) @@ -306,15 +308,14 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, if (!outBuf.Alloc()) return E_OUTOFMEMORY; - CLocalProgress *lps = new CLocalProgress; - CMyComPtr progress = lps; + CMyComPtr2_Create lps; lps->Init(extractCallback, true); CPpmdCpp ppmd(_item.Ver); if (!ppmd.Alloc(_item.MemInMB)) return E_OUTOFMEMORY; - Int32 opRes = NExtract::NOperationResult::kUnsupportedMethod; + opRes = NExtract::NOperationResult::kUnsupportedMethod; if (_item.IsSupported()) { @@ -380,8 +381,7 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, RINOK(inBuf.Res) } - - realOutStream.Release(); +} return extractCallback->SetOperationResult(opRes); } diff --git a/CPP/7zip/Archive/Rar/Rar5Handler.cpp b/CPP/7zip/Archive/Rar/Rar5Handler.cpp index 87d11e7..b786f3e 100644 --- a/CPP/7zip/Archive/Rar/Rar5Handler.cpp +++ b/CPP/7zip/Archive/Rar/Rar5Handler.cpp @@ -17,6 +17,7 @@ #include "../../Common/FilterCoder.h" #include "../../Common/LimitedStreams.h" +#include "../../Common/MethodProps.h" #include "../../Common/ProgressUtils.h" #include "../../Common/RegisterArc.h" #include "../../Common/StreamObjects.h" @@ -28,12 +29,13 @@ #include "../../Crypto/Rar5Aes.h" -#include "../Common/FindSignature.h" -#include "../Common/ItemNameUtils.h" +#include "../../Archive/Common/FindSignature.h" +#include "../../Archive/Common/ItemNameUtils.h" +#include "../../Archive/Common/HandlerOut.h" -#include "../HandlerCont.h" +#include "../../Archive/HandlerCont.h" -#include "RarVol.h" +#include "../../Archive/Rar/RarVol.h" #include "Rar5Handler.h" using namespace NWindows; @@ -48,7 +50,9 @@ static const unsigned kMarkerSize = 8; static const Byte kMarker[kMarkerSize] = { 0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x01, 0 }; -static const size_t kCommentSize_Max = (size_t)1 << 16; +// Comment length is limited to 256 KB in rar-encoder. +// So we use same limitation +static const size_t kCommentSize_Max = (size_t)1 << 18; static const char * const kHostOS[] = @@ -104,34 +108,53 @@ static const char * const g_LinkTypes[] = static const char g_ExtraTimeFlags[] = { 'u', 'M', 'C', 'A', 'n' }; -static unsigned ReadVarInt(const Byte *p, size_t maxSize, UInt64 *val) +static +Z7_NO_INLINE +unsigned ReadVarInt(const Byte *p, size_t maxSize, UInt64 *val_ptr) { - *val = 0; - - for (unsigned i = 0; i < maxSize && i < 10;) + if (maxSize > 10) + maxSize = 10; + UInt64 val = 0; + unsigned i; + for (i = 0; i < maxSize;) { - Byte b = p[i]; - *val |= (UInt64)(b & 0x7F) << (7 * i); + const unsigned b = p[i]; + val |= (UInt64)(b & 0x7F) << (7 * i); i++; if ((b & 0x80) == 0) + { + *val_ptr = val; return i; + } } - return 0; + *val_ptr = 0; +#if 1 + return 0; // 7zip-unrar : strict check of error +#else + return i; // original-unrar : ignore error +#endif +} + + +#define PARSE_VAR_INT(p, size, dest) \ +{ const unsigned num_ = ReadVarInt(p, size, &dest); \ + if (num_ == 0) return false; \ + p += num_; \ + size -= num_; \ } bool CLinkInfo::Parse(const Byte *p, unsigned size) { const Byte *pStart = p; - unsigned num; UInt64 len; - num = ReadVarInt(p, size, &Type); if (num == 0) { return false; } p += num; size -= num; - num = ReadVarInt(p, size, &Flags); if (num == 0) { return false; } p += num; size -= num; - num = ReadVarInt(p, size, &len); if (num == 0) { return false; } p += num; size -= num; + PARSE_VAR_INT(p, size, Type) + PARSE_VAR_INT(p, size, Flags) + PARSE_VAR_INT(p, size, len) if (size != len) return false; NameLen = (unsigned)len; - NameOffset = (unsigned)(p - pStart); + NameOffset = (unsigned)(size_t)(p - pStart); return true; } @@ -174,7 +197,7 @@ int CItem::FindExtra(unsigned extraID, unsigned &recordDataSize) const { UInt64 size; - unsigned num = ReadVarInt(Extra + offset, rem, &size); + const unsigned num = ReadVarInt(Extra + offset, rem, &size); if (num == 0) return -1; offset += num; @@ -185,7 +208,7 @@ int CItem::FindExtra(unsigned extraID, unsigned &recordDataSize) const } { UInt64 id; - unsigned num = ReadVarInt(Extra + offset, rem, &id); + const unsigned num = ReadVarInt(Extra + offset, rem, &id); if (num == 0) return -1; offset += num; @@ -257,17 +280,17 @@ void CItem::PrintInfo(AString &s) const { const Byte *p = Extra + offset; UInt64 flags; - unsigned num = ReadVarInt(p, rem, &flags); + const unsigned num = ReadVarInt(p, rem, &flags); if (num != 0) { - s += ':'; + s.Add_Colon(); for (unsigned i = 0; i < Z7_ARRAY_SIZE(g_ExtraTimeFlags); i++) if ((flags & ((UInt64)1 << i)) != 0) - s += g_ExtraTimeFlags[i]; + s.Add_Char(g_ExtraTimeFlags[i]); flags &= ~(((UInt64)1 << Z7_ARRAY_SIZE(g_ExtraTimeFlags)) - 1); if (flags != 0) { - s += '_'; + s.Add_Char('_'); AddHex64(s, flags); } } @@ -277,20 +300,20 @@ void CItem::PrintInfo(AString &s) const CLinkInfo linkInfo; if (linkInfo.Parse(Extra + offset, (unsigned)rem)) { - s += ':'; + s.Add_Colon(); PrintType(s, g_LinkTypes, Z7_ARRAY_SIZE(g_LinkTypes), linkInfo.Type); UInt64 flags = linkInfo.Flags; if (flags != 0) { - s += ':'; + s.Add_Colon(); if (flags & NLinkFlags::kTargetIsDir) { - s += 'D'; + s.Add_Char('D'); flags &= ~((UInt64)NLinkFlags::kTargetIsDir); } if (flags != 0) { - s += '_'; + s.Add_Char('_'); AddHex64(s, flags); } } @@ -310,19 +333,12 @@ bool CCryptoInfo::Parse(const Byte *p, size_t size) Algo = 0; Flags = 0; Cnt = 0; - - unsigned num = ReadVarInt(p, size, &Algo); - if (num == 0) { return false; } p += num; size -= num; - - num = ReadVarInt(p, size, &Flags); - if (num == 0) { return false; } p += num; size -= num; - + PARSE_VAR_INT(p, size, Algo) + PARSE_VAR_INT(p, size, Flags) if (size > 0) Cnt = p[0]; - if (size != 1 + 16 + 16 + (unsigned)(IsThereCheck() ? 12 : 0)) return false; - return true; } @@ -330,18 +346,14 @@ bool CCryptoInfo::Parse(const Byte *p, size_t size) bool CItem::FindExtra_Version(UInt64 &version) const { unsigned size; - int offset = FindExtra(NExtraID::kVersion, size); + const int offset = FindExtra(NExtraID::kVersion, size); if (offset < 0) return false; const Byte *p = Extra + (unsigned)offset; UInt64 flags; - unsigned num = ReadVarInt(p, size, &flags); - if (num == 0) { return false; } p += num; size -= num; - - num = ReadVarInt(p, size, &version); - if (num == 0) { return false; } p += num; size -= num; - + PARSE_VAR_INT(p, size, flags) + PARSE_VAR_INT(p, size, version) return size == 0; } @@ -407,7 +419,7 @@ bool CItem::GetAltStreamName(AString &name) const { name.Empty(); unsigned size; - int offset = FindExtra(NExtraID::kSubdata, size); + const int offset = FindExtra(NExtraID::kSubdata, size); if (offset < 0) return false; name.SetFrom_CalcLen((const char *)(Extra + (unsigned)offset), size); @@ -420,8 +432,13 @@ class CHash bool _calcCRC; UInt32 _crc; int _blakeOffset; - CBlake2sp _blake; + CAlignedBuffer1 _buf; + // CBlake2sp _blake; + CBlake2sp *BlakeObj() { return (CBlake2sp *)(void *)(Byte *)_buf; } public: + CHash(): + _buf(sizeof(CBlake2sp)) + {} void Init_NoCalc() { @@ -434,17 +451,16 @@ class CHash void Update(const void *data, size_t size); UInt32 GetCRC() const { return CRC_GET_DIGEST(_crc); } - bool Check(const CItem &item, NCrypto::NRar5::CDecoder *cryptoDecoderSpec); + bool Check(const CItem &item, NCrypto::NRar5::CDecoder *cryptoDecoder); }; void CHash::Init(const CItem &item) { _crc = CRC_INIT_VAL; _calcCRC = item.Has_CRC(); - _blakeOffset = item.FindExtra_Blake(); if (_blakeOffset >= 0) - Blake2sp_Init(&_blake); + Blake2sp_Init(BlakeObj()); } void CHash::Update(const void *data, size_t size) @@ -452,30 +468,28 @@ void CHash::Update(const void *data, size_t size) if (_calcCRC) _crc = CrcUpdate(_crc, data, size); if (_blakeOffset >= 0) - Blake2sp_Update(&_blake, (const Byte *)data, size); + Blake2sp_Update(BlakeObj(), (const Byte *)data, size); } -bool CHash::Check(const CItem &item, NCrypto::NRar5::CDecoder *cryptoDecoderSpec) +bool CHash::Check(const CItem &item, NCrypto::NRar5::CDecoder *cryptoDecoder) { if (_calcCRC) { UInt32 crc = GetCRC(); - if (cryptoDecoderSpec) - crc = cryptoDecoderSpec->Hmac_Convert_Crc32(crc); + if (cryptoDecoder) + crc = cryptoDecoder->Hmac_Convert_Crc32(crc); if (crc != item.CRC) return false; } - if (_blakeOffset >= 0) { - Byte digest[BLAKE2S_DIGEST_SIZE]; - Blake2sp_Final(&_blake, digest); - if (cryptoDecoderSpec) - cryptoDecoderSpec->Hmac_Convert_32Bytes(digest); - if (memcmp(digest, &item.Extra[(unsigned)_blakeOffset], BLAKE2S_DIGEST_SIZE) != 0) + UInt32 digest[Z7_BLAKE2S_DIGEST_SIZE / sizeof(UInt32)]; + Blake2sp_Final(BlakeObj(), (Byte *)(void *)digest); + if (cryptoDecoder) + cryptoDecoder->Hmac_Convert_32Bytes((Byte *)(void *)digest); + if (memcmp(digest, item.Extra + (unsigned)_blakeOffset, Z7_BLAKE2S_DIGEST_SIZE) != 0) return false; } - return true; } @@ -484,10 +498,10 @@ Z7_CLASS_IMP_NOQIB_1( COutStreamWithHash , ISequentialOutStream ) + bool _size_Defined; ISequentialOutStream *_stream; UInt64 _pos; UInt64 _size; - bool _size_Defined; Byte *_destBuf; public: CHash _hash; @@ -495,7 +509,7 @@ Z7_CLASS_IMP_NOQIB_1( COutStreamWithHash(): _destBuf(NULL) {} void SetStream(ISequentialOutStream *stream) { _stream = stream; } - void Init(const CItem &item, Byte *destBuf) + void Init(const CItem &item, Byte *destBuf, bool needChecksumCheck) { _size_Defined = false; _size = 0; @@ -507,7 +521,10 @@ Z7_CLASS_IMP_NOQIB_1( _destBuf = destBuf; } _pos = 0; - _hash.Init(item); + if (needChecksumCheck) + _hash.Init(item); + else + _hash.Init_NoCalc(); } UInt64 GetPos() const { return _pos; } }; @@ -518,7 +535,7 @@ Z7_COM7F_IMF(COutStreamWithHash::Write(const void *data, UInt32 size, UInt32 *pr HRESULT result = S_OK; if (_size_Defined) { - UInt64 rem = _size - _pos; + const UInt64 rem = _size - _pos; if (size > rem) size = (UInt32)rem; } @@ -544,8 +561,7 @@ class CInArchive size_t _bufPos; ISequentialInStream *_stream; - NCrypto::NRar5::CDecoder *m_CryptoDecoderSpec; - CMyComPtr m_CryptoDecoder; + CMyComPtr2 m_CryptoDecoder; Z7_CLASS_NO_COPY(CInArchive) @@ -561,6 +577,10 @@ class CInArchive UInt64 StreamStartPosition; UInt64 Position; + size_t Get_Buf_RemainSize() const { return _bufSize - _bufPos; } + bool Is_Buf_Finished() const { return _bufPos == _bufSize; } + const Byte *Get_Buf_Data() const { return _buf + _bufPos; } + void Move_BufPos(size_t num) { _bufPos += num; } bool ReadVar(UInt64 &val); struct CHeader @@ -585,7 +605,7 @@ class CInArchive }; -static HRESULT MySetPassword(ICryptoGetTextPassword *getTextPassword, NCrypto::NRar5::CDecoder *cryptoDecoderSpec) +static HRESULT MySetPassword(ICryptoGetTextPassword *getTextPassword, NCrypto::NRar5::CDecoder *cryptoDecoder) { CMyComBSTR_Wipe password; RINOK(getTextPassword->CryptoGetTextPassword(&password)) @@ -596,15 +616,15 @@ static HRESULT MySetPassword(ICryptoGetTextPassword *getTextPassword, NCrypto::N if (unicode.Len() > kPasswordLen_MAX) unicode.DeleteFrom(kPasswordLen_MAX); ConvertUnicodeToUTF8(unicode, utf8); - cryptoDecoderSpec->SetPassword((const Byte *)(const char *)utf8, utf8.Len()); + cryptoDecoder->SetPassword((const Byte *)(const char *)utf8, utf8.Len()); return S_OK; } bool CInArchive::ReadVar(UInt64 &val) { - unsigned offset = ReadVarInt(_buf + _bufPos, _bufSize - _bufPos, &val); - _bufPos += offset; + const unsigned offset = ReadVarInt(Get_Buf_Data(), Get_Buf_RemainSize(), &val); + Move_BufPos(offset); return (offset != 0); } @@ -627,61 +647,63 @@ HRESULT CInArchive::ReadBlockHeader(CHeader &h) h.ExtraSize = 0; h.DataSize = 0; - const unsigned kStartSize = 4 + 3; - const unsigned kBufSize = AES_BLOCK_SIZE + AES_BLOCK_SIZE; // must be >= kStartSize; - Byte buf[kBufSize]; + Byte buf[AES_BLOCK_SIZE]; unsigned filled; if (m_CryptoMode) { - RINOK(ReadStream_Check(buf, kBufSize)) - memcpy(m_CryptoDecoderSpec->_iv, buf, AES_BLOCK_SIZE); - RINOK(m_CryptoDecoderSpec->Init()) - - _buf.AllocAtLeast(1 << 12); + _buf.AllocAtLeast(1 << 12); // at least (AES_BLOCK_SIZE * 2) if (!(Byte *)_buf) return E_OUTOFMEMORY; - - memcpy(_buf, buf + AES_BLOCK_SIZE, AES_BLOCK_SIZE); - if (m_CryptoDecoder->Filter(_buf, AES_BLOCK_SIZE) != AES_BLOCK_SIZE) + RINOK(ReadStream_Check(_buf, AES_BLOCK_SIZE * 2)) + memcpy(m_CryptoDecoder->_iv, _buf, AES_BLOCK_SIZE); + RINOK(m_CryptoDecoder->Init()) + if (m_CryptoDecoder->Filter(_buf + AES_BLOCK_SIZE, AES_BLOCK_SIZE) != AES_BLOCK_SIZE) return E_FAIL; - memcpy(buf, _buf, AES_BLOCK_SIZE); + memcpy(buf, _buf + AES_BLOCK_SIZE, AES_BLOCK_SIZE); filled = AES_BLOCK_SIZE; } else { + const unsigned kStartSize = 4 + 3; RINOK(ReadStream_Check(buf, kStartSize)) filled = kStartSize; } - UInt64 val; - unsigned offset = ReadVarInt(buf + 4, 3, &val); - if (offset == 0) - return S_FALSE; { + UInt64 val; + unsigned offset = ReadVarInt(buf + 4, 3, &val); + if (offset == 0) + return S_FALSE; size_t size = (size_t)val; - _bufPos = (4 + offset); - _bufSize = _bufPos + size; if (size < 2) return S_FALSE; - } - - size_t allocSize = _bufSize; - if (m_CryptoMode) - allocSize = (allocSize + AES_BLOCK_SIZE - 1) & ~(size_t)(AES_BLOCK_SIZE - 1); - _buf.AllocAtLeast(allocSize); - if (!(Byte *)_buf) - return E_OUTOFMEMORY; - - memcpy(_buf, buf, filled); - - size_t rem = allocSize - filled; - AddToSeekValue(allocSize + (m_CryptoMode ? AES_BLOCK_SIZE : 0)); - RINOK(ReadStream_Check(_buf + filled, rem)) - if (m_CryptoMode) - { - if (m_CryptoDecoder->Filter(_buf + filled, (UInt32)rem) != rem) - return E_FAIL; + offset += 4; + _bufPos = offset; + size += offset; + _bufSize = size; + if (m_CryptoMode) + size = (size + AES_BLOCK_SIZE - 1) & ~(size_t)(AES_BLOCK_SIZE - 1); + _buf.AllocAtLeast(size); + if (!(Byte *)_buf) + return E_OUTOFMEMORY; + memcpy(_buf, buf, filled); + const size_t rem = size - filled; + AddToSeekValue(size + (m_CryptoMode ? AES_BLOCK_SIZE : 0)); + RINOK(ReadStream_Check(_buf + filled, rem)) + if (m_CryptoMode) + { + if (m_CryptoDecoder->Filter(_buf + filled, (UInt32)rem) != rem) + return E_FAIL; +#if 1 + // optional 7zip-unrar check : remainder must contain zeros. + const size_t pad = size - _bufSize; + const Byte *p = _buf + _bufSize; + for (size_t i = 0; i < pad; i++) + if (p[i]) + return S_FALSE; +#endif + } } if (CrcCalc(_buf + 4, _bufSize - 4) != Get32(buf)) @@ -695,7 +717,7 @@ HRESULT CInArchive::ReadBlockHeader(CHeader &h) UInt64 extraSize; if (!ReadVar(extraSize)) return S_FALSE; - if (extraSize > _bufSize) + if (extraSize >= (1u << 21)) return S_FALSE; h.ExtraSize = (size_t)extraSize; } @@ -706,85 +728,117 @@ HRESULT CInArchive::ReadBlockHeader(CHeader &h) return S_FALSE; } + if (h.ExtraSize > Get_Buf_RemainSize()) + return S_FALSE; return S_OK; } -/* -int CInArcInfo::FindExtra(unsigned extraID, unsigned &recordDataSize) const +bool CInArcInfo::CLocator::Parse(const Byte *p, size_t size) { - recordDataSize = 0; - size_t offset = 0; + Flags = 0; + QuickOpen = 0; + Recovery = 0; - for (;;) + PARSE_VAR_INT(p, size, Flags) + + if (Is_QuickOpen()) { - size_t rem = Extra.Size() - offset; - if (rem == 0) - return -1; - + PARSE_VAR_INT(p, size, QuickOpen) + } + if (Is_Recovery()) + { + PARSE_VAR_INT(p, size, Recovery) + } +#if 0 + // another records are possible in future rar formats. + if (size != 0) + return false; +#endif + return true; +} + + +bool CInArcInfo::CMetadata::Parse(const Byte *p, size_t size) +{ + PARSE_VAR_INT(p, size, Flags) + if (Flags & NMetadataFlags::kArcName) + { + UInt64 nameLen; + PARSE_VAR_INT(p, size, nameLen) + if (nameLen > size) + return false; + ArcName.SetFrom_CalcLen((const char *)(const void *)p, (unsigned)nameLen); + p += (size_t)nameLen; + size -= (size_t)nameLen; + } + if (Flags & NMetadataFlags::kCTime) + { + if ((Flags & NMetadataFlags::kUnixTime) && + (Flags & NMetadataFlags::kNanoSec) == 0) { - UInt64 size; - unsigned num = ReadVarInt(Extra + offset, rem, &size); - if (num == 0) - return -1; - offset += num; - rem -= num; - if (size > rem) - return -1; - rem = (size_t)size; + if (size < 4) + return false; + CTime = GetUi32(p); + p += 4; + size -= 4; } + else { - UInt64 id; - unsigned num = ReadVarInt(Extra + offset, rem, &id); - if (num == 0) - return -1; - offset += num; - rem -= num; - - if (id == extraID) - { - recordDataSize = (unsigned)rem; - return (int)offset; - } - - offset += rem; + if (size < 8) + return false; + CTime = GetUi64(p); + p += 8; + size -= 8; } } +#if 0 + // another records are possible in future rar formats. + if (size != 0) + return false; +#endif + return true; } -bool CInArcInfo::FindExtra_Locator(CLocator &locator) const +bool CInArcInfo::ParseExtra(const Byte *p, size_t size) { - locator.Flags = 0; - locator.QuickOpen = 0; - locator.Recovery = 0; - - unsigned size; - int offset = FindExtra(kArcExtraRecordType_Locator, size); - if (offset < 0) - return false; - const Byte *p = Extra + (unsigned)offset; - - unsigned num; - - num = ReadVarInt(p, size, &locator.Flags); - if (num == 0) return false; p += num; size -= num; - - if (locator.Is_QuickOpen()) - { - num = ReadVarInt(p, size, &locator.QuickOpen); - if (num == 0) return false; p += num; size -= num; - } - - if (locator.Is_Recovery()) + for (;;) { - num = ReadVarInt(p, size, &locator.Recovery); - if (num == 0) return false; p += num; size -= num; + if (size == 0) + return true; + UInt64 recSize64, id; + PARSE_VAR_INT(p, size, recSize64) + if (recSize64 > size) + return false; + size_t recSize = (size_t)recSize64; + size -= recSize; + // READ_VAR_INT(p, recSize, recSize) + { + const unsigned num = ReadVarInt(p, recSize, &id); + if (num == 0) + return false; + p += num; + recSize -= num; + } + if (id == kArcExtraRecordType_Metadata) + { + Metadata_Defined = true; + if (!Metadata.Parse(p, recSize)) + Metadata_Error = true; + } + else if (id == kArcExtraRecordType_Locator) + { + Locator_Defined = true; + if (!Locator.Parse(p, recSize)) + Locator_Error = true; + } + else + UnknownExtraRecord = true; + p += recSize; } - - return true; } -*/ + HRESULT CInArchive::Open(IInStream *stream, const UInt64 *searchHeaderSizeLimit, ICryptoGetTextPassword *getTextPassword, @@ -830,26 +884,16 @@ HRESULT CInArchive::Open(IInStream *stream, const UInt64 *searchHeaderSizeLimit, IsArc = true; if (!getTextPassword) return E_NOTIMPL; - m_CryptoMode = true; - - if (!m_CryptoDecoder) - { - m_CryptoDecoderSpec = new NCrypto::NRar5::CDecoder; - m_CryptoDecoder = m_CryptoDecoderSpec; - } - - RINOK(m_CryptoDecoderSpec->SetDecoderProps( - _buf + _bufPos, (unsigned)(_bufSize - _bufPos), false, false)) - - RINOK(MySetPassword(getTextPassword, m_CryptoDecoderSpec)) - - if (!m_CryptoDecoderSpec->CalcKey_and_CheckPassword()) + m_CryptoDecoder.Create_if_Empty(); + RINOK(m_CryptoDecoder->SetDecoderProps( + Get_Buf_Data(), (unsigned)Get_Buf_RemainSize(), false, false)) + RINOK(MySetPassword(getTextPassword, m_CryptoDecoder.ClsPtr())) + if (!m_CryptoDecoder->CalcKey_and_CheckPassword()) { WrongPassword = True; return S_FALSE; } - RINOK(ReadBlockHeader(h)) } @@ -866,42 +910,29 @@ HRESULT CInArchive::Open(IInStream *stream, const UInt64 *searchHeaderSizeLimit, if (!ReadVar(info.VolNumber)) return S_FALSE; - if (h.ExtraSize != 0) + if (h.ExtraSize != Get_Buf_RemainSize()) + return S_FALSE; + if (h.ExtraSize) { - if (_bufSize - _bufPos < h.ExtraSize) - return S_FALSE; - /* - info.Extra.Alloc(h.ExtraSize); - memcpy(info.Extra, _buf + _bufPos, h.ExtraSize); - */ - _bufPos += h.ExtraSize; - - /* - CInArcInfo::CLocator locator; - if (info.FindExtra_Locator(locator)) - locator.Flags = locator.Flags; - */ + if (!info.ParseExtra(Get_Buf_Data(), h.ExtraSize)) + info.Extra_Error = true; } - - if (_bufPos != _bufSize) - return S_FALSE; - return S_OK; } bool CInArchive::ReadFileHeader(const CHeader &header, CItem &item) { - item.UnixMTime = 0; - item.CRC = 0; - item.Flags = 0; - item.CommonFlags = (UInt32)header.Flags; item.PackSize = header.DataSize; + item.UnixMTime = 0; + item.CRC = 0; - UInt64 flags64; - if (!ReadVar(flags64)) return false; - item.Flags = (UInt32)flags64; + { + UInt64 flags64; + if (!ReadVar(flags64)) return false; + item.Flags = (UInt32)flags64; + } if (!ReadVar(item.Size)) return false; @@ -910,23 +941,20 @@ bool CInArchive::ReadFileHeader(const CHeader &header, CItem &item) if (!ReadVar(attrib)) return false; item.Attrib = (UInt32)attrib; } - if (item.Has_UnixMTime()) { - if (_bufSize - _bufPos < 4) + if (Get_Buf_RemainSize() < 4) return false; - item.UnixMTime = Get32(_buf + _bufPos); - _bufPos += 4; + item.UnixMTime = Get32(Get_Buf_Data()); + Move_BufPos(4); } - if (item.Has_CRC()) { - if (_bufSize - _bufPos < 4) + if (Get_Buf_RemainSize() < 4) return false; - item.CRC = Get32(_buf + _bufPos); - _bufPos += 4; + item.CRC = Get32(Get_Buf_Data()); + Move_BufPos(4); } - { UInt64 method; if (!ReadVar(method)) return false; @@ -938,25 +966,24 @@ bool CInArchive::ReadFileHeader(const CHeader &header, CItem &item) { UInt64 len; if (!ReadVar(len)) return false; - if (len > _bufSize - _bufPos) + if (len > Get_Buf_RemainSize()) return false; - item.Name.SetFrom_CalcLen((const char *)(_buf + _bufPos), (unsigned)len); - _bufPos += (unsigned)len; + item.Name.SetFrom_CalcLen((const char *)Get_Buf_Data(), (unsigned)len); + Move_BufPos((size_t)len); } item.Extra.Free(); - size_t extraSize = header.ExtraSize; + const size_t extraSize = header.ExtraSize; if (extraSize != 0) { - if (_bufSize - _bufPos < extraSize) + if (Get_Buf_RemainSize() < extraSize) return false; item.Extra.Alloc(extraSize); - memcpy(item.Extra, _buf + _bufPos, extraSize); - _bufPos += extraSize; + memcpy(item.Extra, Get_Buf_Data(), extraSize); + Move_BufPos(extraSize); } - - return (_bufPos == _bufSize); + return Is_Buf_Finished(); } @@ -975,77 +1002,59 @@ struct CLinkFile struct CUnpacker { - NCompress::CCopyCoder *copyCoderSpec; - CMyComPtr copyCoder; - + CMyComPtr2 copyCoder; CMyComPtr LzCoders[2]; bool SolidAllowed; - + bool NeedCrc; CFilterCoder *filterStreamSpec; CMyComPtr filterStream; - - NCrypto::NRar5::CDecoder *cryptoDecoderSpec; - CMyComPtr cryptoDecoder; - + CMyComPtr2 cryptoDecoder; CMyComPtr getTextPassword; - - COutStreamWithHash *outStreamSpec; - CMyComPtr outStream; + CMyComPtr2 outStream; CByteBuffer _tempBuf; - CLinkFile *linkFile; - CUnpacker(): linkFile(NULL) { SolidAllowed = false; } + CUnpacker(): linkFile(NULL) { SolidAllowed = false; NeedCrc = true; } - HRESULT Create(DECL_EXTERNAL_CODECS_LOC_VARS const CItem &item, bool isSolid, bool &wrongPassword); + HRESULT Create(DECL_EXTERNAL_CODECS_LOC_VARS + const CItem &item, bool isSolid, bool &wrongPassword); HRESULT Code(const CItem &item, const CItem &lastItem, UInt64 packSize, ISequentialInStream *inStream, ISequentialOutStream *outStream, ICompressProgressInfo *progress, bool &isCrcOK); - HRESULT DecodeToBuf(DECL_EXTERNAL_CODECS_LOC_VARS const CItem &item, UInt64 packSize, ISequentialInStream *inStream, CByteBuffer &buffer); + HRESULT DecodeToBuf(DECL_EXTERNAL_CODECS_LOC_VARS + const CItem &item, UInt64 packSize, ISequentialInStream *inStream, CByteBuffer &buffer); }; static const unsigned kLzMethodMax = 5; -HRESULT CUnpacker::Create(DECL_EXTERNAL_CODECS_LOC_VARS const CItem &item, bool isSolid, bool &wrongPassword) +HRESULT CUnpacker::Create(DECL_EXTERNAL_CODECS_LOC_VARS + const CItem &item, bool isSolid, bool &wrongPassword) { wrongPassword = false; - if (item.GetAlgoVersion() != 0) + if (item.Get_AlgoVersion_RawBits() > 1) return E_NOTIMPL; - if (!outStream) - { - outStreamSpec = new COutStreamWithHash; - outStream = outStreamSpec; - } + outStream.Create_if_Empty(); - unsigned method = item.GetMethod(); + const unsigned method = item.Get_Method(); if (method == 0) - { - if (!copyCoder) - { - copyCoderSpec = new NCompress::CCopyCoder; - copyCoder = copyCoderSpec; - } - } + copyCoder.Create_if_Empty(); else { if (method > kLzMethodMax) return E_NOTIMPL; - /* if (item.IsSplitBefore()) return S_FALSE; */ - - int lzIndex = item.IsService() ? 1 : 0; + const unsigned lzIndex = item.IsService() ? 1 : 0; CMyComPtr &lzCoder = LzCoders[lzIndex]; - if (!lzCoder) { const UInt32 methodID = 0x40305; @@ -1057,12 +1066,19 @@ HRESULT CUnpacker::Create(DECL_EXTERNAL_CODECS_LOC_VARS const CItem &item, bool CMyComPtr csdp; RINOK(lzCoder.QueryInterface(IID_ICompressSetDecoderProperties2, &csdp)) - Byte props[2] = { (Byte)(item.GetDictSize()), (Byte)(isSolid ? 1 : 0) }; + const unsigned ver = item.Get_AlgoVersion_HuffRev(); + if (ver > 1) + return E_NOTIMPL; + const Byte props[2] = + { + (Byte)item.Get_DictSize_Main(), + (Byte)((item.Get_DictSize_Frac() << 3) + (ver << 1) + (isSolid ? 1 : 0)) + }; RINOK(csdp->SetDecoderProperties2(props, 2)) } unsigned cryptoSize = 0; - int cryptoOffset = item.FindExtra(NExtraID::kCrypto, cryptoSize); + const int cryptoOffset = item.FindExtra(NExtraID::kCrypto, cryptoSize); if (cryptoOffset >= 0) { @@ -1072,13 +1088,9 @@ HRESULT CUnpacker::Create(DECL_EXTERNAL_CODECS_LOC_VARS const CItem &item, bool filterStream = filterStreamSpec; } - if (!cryptoDecoder) - { - cryptoDecoderSpec = new NCrypto::NRar5::CDecoder; - cryptoDecoder = cryptoDecoderSpec; - } + cryptoDecoder.Create_if_Empty(); - RINOK(cryptoDecoderSpec->SetDecoderProps(item.Extra + (unsigned)cryptoOffset, cryptoSize, true, item.IsService())) + RINOK(cryptoDecoder->SetDecoderProps(item.Extra + (unsigned)cryptoOffset, cryptoSize, true, item.IsService())) if (!getTextPassword) { @@ -1086,9 +1098,9 @@ HRESULT CUnpacker::Create(DECL_EXTERNAL_CODECS_LOC_VARS const CItem &item, bool return E_NOTIMPL; } - RINOK(MySetPassword(getTextPassword, cryptoDecoderSpec)) + RINOK(MySetPassword(getTextPassword, cryptoDecoder.ClsPtr())) - if (!cryptoDecoderSpec->CalcKey_and_CheckPassword()) + if (!cryptoDecoder->CalcKey_and_CheckPassword()) wrongPassword = True; } @@ -1102,15 +1114,15 @@ HRESULT CUnpacker::Code(const CItem &item, const CItem &lastItem, UInt64 packSiz { isCrcOK = true; - unsigned method = item.GetMethod(); + const unsigned method = item.Get_Method(); if (method > kLzMethodMax) return E_NOTIMPL; - bool needBuf = (linkFile && linkFile->NumLinks != 0); + const bool needBuf = (linkFile && linkFile->NumLinks != 0); if (needBuf && !lastItem.Is_UnknownSize()) { - size_t dataSize = (size_t)lastItem.Size; + const size_t dataSize = (size_t)lastItem.Size; if (dataSize != lastItem.Size) return E_NOTIMPL; linkFile->Data.Alloc(dataSize); @@ -1130,10 +1142,12 @@ HRESULT CUnpacker::Code(const CItem &item, const CItem &lastItem, UInt64 packSiz else inStream = volsInStream; - ICompressCoder *commonCoder = (method == 0) ? copyCoder : LzCoders[item.IsService() ? 1 : 0]; + ICompressCoder *commonCoder = (method == 0) ? + copyCoder.Interface() : + LzCoders[item.IsService() ? 1 : 0].Interface(); - outStreamSpec->SetStream(realOutStream); - outStreamSpec->Init(lastItem, (needBuf ? (Byte *)linkFile->Data : NULL)); + outStream->SetStream(realOutStream); + outStream->Init(lastItem, (needBuf ? (Byte *)linkFile->Data : NULL), NeedCrc); HRESULT res = S_OK; if (packSize != 0 || lastItem.Is_UnknownSize() || lastItem.Size != 0) @@ -1151,25 +1165,24 @@ HRESULT CUnpacker::Code(const CItem &item, const CItem &lastItem, UInt64 packSiz if (isCryptoMode) filterStreamSpec->ReleaseInStream(); - UInt64 processedSize = outStreamSpec->GetPos(); + const UInt64 processedSize = outStream->GetPos(); if (res == S_OK && !lastItem.Is_UnknownSize() && processedSize != lastItem.Size) res = S_FALSE; // if (res == S_OK) { unsigned cryptoSize = 0; - int cryptoOffset = lastItem.FindExtra(NExtraID::kCrypto, cryptoSize); + const int cryptoOffset = lastItem.FindExtra(NExtraID::kCrypto, cryptoSize); NCrypto::NRar5::CDecoder *crypto = NULL; - if (cryptoOffset >= 0) { CCryptoInfo cryptoInfo; if (cryptoInfo.Parse(lastItem.Extra + (unsigned)cryptoOffset, cryptoSize)) if (cryptoInfo.UseMAC()) - crypto = cryptoDecoderSpec; + crypto = cryptoDecoder.ClsPtr(); } - - isCrcOK = outStreamSpec->_hash.Check(lastItem, crypto); + if (NeedCrc) + isCrcOK = outStream->_hash.Check(lastItem, crypto); } if (linkFile) @@ -1186,12 +1199,14 @@ HRESULT CUnpacker::Code(const CItem &item, const CItem &lastItem, UInt64 packSiz } -HRESULT CUnpacker::DecodeToBuf(DECL_EXTERNAL_CODECS_LOC_VARS const CItem &item, UInt64 packSize, ISequentialInStream *inStream, CByteBuffer &buffer) +HRESULT CUnpacker::DecodeToBuf(DECL_EXTERNAL_CODECS_LOC_VARS + const CItem &item, UInt64 packSize, + ISequentialInStream *inStream, + CByteBuffer &buffer) { - CBufPtrSeqOutStream *outSpec = new CBufPtrSeqOutStream; - CMyComPtr out = outSpec; + CMyComPtr2_Create out; _tempBuf.AllocAtLeast((size_t)item.Size); - outSpec->Init(_tempBuf, (size_t)item.Size); + out->Init(_tempBuf, (size_t)item.Size); bool wrongPassword; @@ -1205,16 +1220,15 @@ HRESULT CUnpacker::DecodeToBuf(DECL_EXTERNAL_CODECS_LOC_VARS const CItem &item, if (wrongPassword) return S_FALSE; - CLimitedSequentialInStream *limitedStreamSpec = new CLimitedSequentialInStream; - CMyComPtr limitedStream(limitedStreamSpec); - limitedStreamSpec->SetStream(inStream); - limitedStreamSpec->Init(packSize); + CMyComPtr2_Create limitedStream; + limitedStream->SetStream(inStream); + limitedStream->Init(packSize); bool crcOK = true; res = Code(item, item, packSize, limitedStream, out, NULL, crcOK); if (res == S_OK) { - if (!crcOK || outSpec->GetPos() != item.Size) + if (!crcOK || out->GetPos() != item.Size) res = S_FALSE; else buffer.CopyFrom(_tempBuf, (size_t)item.Size); @@ -1241,7 +1255,9 @@ struct CTempBuf HRESULT Decode(DECL_EXTERNAL_CODECS_LOC_VARS const CItem &item, - ISequentialInStream *inStream, CUnpacker &unpacker, CByteBuffer &destBuf); + ISequentialInStream *inStream, + CUnpacker &unpacker, + CByteBuffer &destBuf); }; @@ -1294,9 +1310,8 @@ HRESULT CTempBuf::Decode(DECL_EXTERNAL_CODECS_LOC_VARS } else { - CBufInStream *bufInStreamSpec = new CBufInStream; - CMyComPtr bufInStream = bufInStreamSpec; - bufInStreamSpec->Init(_buf, _offset); + CMyComPtr2_Create bufInStream; + bufInStream->Init(_buf, _offset); RINOK(unpacker.DecodeToBuf(EXTERNAL_CODECS_LOC_VARS item, _offset, bufInStream, destBuf)) } @@ -1318,6 +1333,7 @@ static const Byte kProps[] = kpidCTime, kpidATime, kpidAttrib, + // kpidPosixAttrib, // for debug kpidIsAltStream, kpidEncrypted, @@ -1340,12 +1356,15 @@ static const Byte kArcProps[] = { kpidTotalPhySize, kpidCharacts, + kpidEncrypted, kpidSolid, kpidNumBlocks, - kpidEncrypted, + kpidMethod, kpidIsVolume, kpidVolumeIndex, kpidNumVolumes, + kpidName, + kpidCTime, kpidComment }; @@ -1368,6 +1387,17 @@ UInt64 CHandler::GetPackSize(unsigned refIndex) const } } +static char *PrintDictSize(char *s, UInt64 w) +{ + char c = 'K'; w >>= 10; + if ((w & ((1 << 10) - 1)) == 0) { c = 'M'; w >>= 10; + if ((w & ((1 << 10) - 1)) == 0) { c = 'G'; w >>= 10; }} + s = ConvertUInt64ToString(w, s); + *s++ = c; + *s = 0; + return s; +} + Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) { @@ -1385,10 +1415,69 @@ Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) case kpidSolid: if (arcInfo) prop = arcInfo->IsSolid(); break; case kpidCharacts: { - if (!_arcs.IsEmpty()) + AString s; + if (arcInfo) { - FLAGS_TO_PROP(k_ArcFlags, (UInt32)arcInfo->Flags, prop); + s = FlagsToString(k_ArcFlags, Z7_ARRAY_SIZE(k_ArcFlags), (UInt32)arcInfo->Flags); + if (arcInfo->Extra_Error) + s.Add_OptSpaced("Extra-ERROR"); + if (arcInfo->UnsupportedFeature) + s.Add_OptSpaced("unsupported-feature"); + if (arcInfo->Metadata_Defined) + { + s.Add_OptSpaced("Metadata"); + if (arcInfo->Metadata_Error) + s += "-ERROR"; + else + { + if (arcInfo->Metadata.Flags & NMetadataFlags::kArcName) + s.Add_OptSpaced("arc-name"); + if (arcInfo->Metadata.Flags & NMetadataFlags::kCTime) + { + s.Add_OptSpaced("ctime-"); + s += + (arcInfo->Metadata.Flags & NMetadataFlags::kUnixTime) ? + (arcInfo->Metadata.Flags & NMetadataFlags::kNanoSec) ? + "1ns" : "1s" : "win"; + } + } + } + if (arcInfo->Locator_Defined) + { + s.Add_OptSpaced("Locator"); + if (arcInfo->Locator_Error) + s += "-ERROR"; + else + { + if (arcInfo->Locator.Is_QuickOpen()) + { + s.Add_OptSpaced("QuickOpen:"); + s.Add_UInt64(arcInfo->Locator.QuickOpen); + } + if (arcInfo->Locator.Is_Recovery()) + { + s += "Recovery:"; + s.Add_UInt64(arcInfo->Locator.Recovery); + } + } + } + if (arcInfo->UnknownExtraRecord) + s.Add_OptSpaced("Unknown-Extra-Record"); + } + if (_comment_WasUsedInArc) + { + s.Add_OptSpaced("Comment"); + // s.Add_UInt32((UInt32)_comment.Size()); + } + // + if (_acls.Size() != 0) + { + s.Add_OptSpaced("ACL"); + // s.Add_UInt32(_acls.Size()); + } + if (!s.IsEmpty()) + prop = s; break; } case kpidEncrypted: if (arcInfo) prop = arcInfo->IsEncrypted; break; // it's for encrypted names. @@ -1415,13 +1504,55 @@ Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) break; } + case kpidName: + if (arcInfo) + if (!arcInfo->Metadata_Error + && !arcInfo->Metadata.ArcName.IsEmpty()) + { + UString s; + if (ConvertUTF8ToUnicode(arcInfo->Metadata.ArcName, s)) + prop = s; + } + break; + + case kpidCTime: + if (arcInfo) + if (!arcInfo->Metadata_Error + && (arcInfo->Metadata.Flags & NMetadataFlags::kCTime)) + { + const UInt64 ct = arcInfo->Metadata.CTime; + if (arcInfo->Metadata.Flags & NMetadataFlags::kUnixTime) + { + if (arcInfo->Metadata.Flags & NMetadataFlags::kNanoSec) + { + const UInt64 sec = ct / 1000000000; + const UInt64 ns = ct % 1000000000; + UInt64 wt = NTime::UnixTime64_To_FileTime64((Int64)sec); + wt += ns / 100; + const unsigned ns100 = (unsigned)(ns % 100); + FILETIME ft; + ft.dwLowDateTime = (DWORD)(UInt32)wt; + ft.dwHighDateTime = (DWORD)(UInt32)(wt >> 32); + prop.SetAsTimeFrom_FT_Prec_Ns100(ft, k_PropVar_TimePrec_1ns, ns100); + } + else + { + const UInt64 wt = NTime::UnixTime64_To_FileTime64((Int64)ct); + prop.SetAsTimeFrom_Ft64_Prec(wt, k_PropVar_TimePrec_Unix); + } + } + else + prop.SetAsTimeFrom_Ft64_Prec(ct, k_PropVar_TimePrec_100ns); + } + break; + case kpidComment: { // if (!_arcs.IsEmpty()) { // const CArc &arc = _arcs[0]; const CByteBuffer &cmt = _comment; - if (cmt.Size() != 0 && cmt.Size() < (1 << 16)) + if (cmt.Size() != 0 /* && cmt.Size() < (1 << 16) */) { AString s; s.SetFrom_CalcLen((const char *)(const Byte *)cmt, (unsigned)cmt.Size()); @@ -1435,11 +1566,48 @@ Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) case kpidNumBlocks: { - UInt32 numBlocks = 0; - FOR_VECTOR (i, _refs) - if (!_items[_refs[i].Item].IsSolid()) - numBlocks++; - prop = (UInt32)numBlocks; + prop = (UInt32)_numBlocks; + break; + } + + case kpidMethod: + { + AString s; + + UInt64 algo = _algo_Mask; + for (unsigned v = 0; algo != 0; v++, algo >>= 1) + { + if ((algo & 1) == 0) + continue; + s.Add_OptSpaced("v"); + s.Add_UInt32(v + 6); + if (v < Z7_ARRAY_SIZE(_methodMasks)) + { + const UInt64 dict = _dictMaxSizes[v]; + if (dict) + { + char temp[24]; + temp[0] = ':'; + PrintDictSize(temp + 1, dict); + s += temp; + } + unsigned method = _methodMasks[v]; + for (unsigned m = 0; method; m++, method >>= 1) + { + if ((method & 1) == 0) + continue; + s += ":m"; + s.Add_UInt32(m); + } + } + } + if (_rar5comapt_mask & 2) + { + s += ":c"; + if (_rar5comapt_mask & 1) + s.Add_Char('n'); + } + prop = s; break; } @@ -1459,6 +1627,10 @@ Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) UInt32 v = _errorFlags; if (!_isArc) v |= kpv_ErrorFlags_IsNotArc; + if (_error_in_ACL) + v |= kpv_ErrorFlags_HeadersError; + if (_split_Error) + v |= kpv_ErrorFlags_HeadersError; prop = v; break; } @@ -1480,7 +1652,7 @@ Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) AString s ("part"); UInt32 v = (UInt32)arcInfo->GetVolIndex() + 1; if (v < 10) - s += '0'; + s.Add_Char('0'); s.Add_UInt32(v); s += ".rar"; prop = s; @@ -1500,7 +1672,7 @@ Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) Z7_COM7F_IMF(CHandler::GetNumberOfItems(UInt32 *numItems)) { - *numItems = _refs.Size(); + *numItems = (UInt32)_refs.Size(); return S_OK; } @@ -1571,13 +1743,21 @@ Z7_COM7F_IMF(CHandler::GetRawProp(UInt32 index, PROPID propID, const void **data if (propID == kpidChecksum) { - int hashRecOffset = item.FindExtra_Blake(); + const int hashRecOffset = item.FindExtra_Blake(); if (hashRecOffset >= 0) { - *dataSize = BLAKE2S_DIGEST_SIZE; + *dataSize = Z7_BLAKE2S_DIGEST_SIZE; *propType = NPropDataType::kRaw; - *data = &item.Extra[hashRecOffset]; + *data = item.Extra + (unsigned)hashRecOffset; } + /* + else if (item.Has_CRC() && item.IsEncrypted()) + { + *dataSize = 4; + *propType = NPropDataType::kRaw; + *data = &item->CRC; // we must show same value for big/little endian here + } + */ return S_OK; } @@ -1594,6 +1774,7 @@ static void TimeRecordToProp(const CItem &item, unsigned stampIndex, NCOM::CProp const Byte *p = item.Extra + (unsigned)offset; UInt64 flags; + // PARSE_VAR_INT(p, size, flags) { const unsigned num = ReadVarInt(p, size, &flags); if (num == 0) @@ -1684,7 +1865,7 @@ Z7_COM7F_IMF(CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val AString name; item.GetAltStreamName(name); if (name[0] != ':') - s += ':'; + s.Add_Colon(); s += name; ConvertUTF8ToUnicode(s, unicodeName); } @@ -1714,7 +1895,7 @@ Z7_COM7F_IMF(CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val case kpidIsDir: prop = item.IsDir(); break; case kpidSize: if (!lastItem.Is_UnknownSize()) prop = lastItem.Size; break; - case kpidPackSize: prop = GetPackSize(index); break; + case kpidPackSize: prop = GetPackSize((unsigned)index); break; case kpidMTime: { @@ -1757,6 +1938,11 @@ Z7_COM7F_IMF(CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val case kpidCopyLink: item.Link_to_Prop(NLinkType::kFileCopy, prop); break; case kpidAttrib: prop = item.GetWinAttrib(); break; + case kpidPosixAttrib: + if (item.HostOS == kHost_Unix) + prop = (UInt32)item.Attrib; + break; + case kpidEncrypted: prop = item.IsEncrypted(); break; case kpidSolid: prop = item.IsSolid(); break; @@ -1777,7 +1963,9 @@ Z7_COM7F_IMF(CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val case kpidCRC: { const CItem *item2 = (lastItem.IsSplitAfter() ? &item : &lastItem); - if (item2->Has_CRC()) + // we don't want to show crc for encrypted file here, + // because crc is also encrrypted. + if (item2->Has_CRC() && !item2->IsEncrypted()) prop = item2->CRC; break; } @@ -1785,56 +1973,61 @@ Z7_COM7F_IMF(CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val case kpidMethod: { char temp[128]; - unsigned algo = item.GetAlgoVersion(); + const unsigned algo = item.Get_AlgoVersion_RawBits(); char *s = temp; - if (algo != 0) + // if (algo != 0) { - ConvertUInt32ToString(algo, s); - s += MyStringLen(s); + *s++ = 'v'; + s = ConvertUInt32ToString((UInt32)algo + 6, s); + if (item.Is_Rar5_Compat()) + *s++ = 'c'; *s++ = ':'; } - unsigned m = item.GetMethod(); { - s[0] = 'm'; - s[1] = (char)(m + '0'); - s[2] = 0; + const unsigned m = item.Get_Method(); + *s++ = 'm'; + *s++ = (char)(m + '0'); if (!item.IsDir()) { - s[2] = ':'; - ConvertUInt32ToString(item.GetDictSize() + 17, s + 3); + *s++ = ':'; + const unsigned dictMain = item.Get_DictSize_Main(); + const unsigned frac = item.Get_DictSize_Frac(); + /* + if (frac == 0 && algo == 0) + s = ConvertUInt32ToString(dictMain + 17, s); + else + */ + s = PrintDictSize(s, (UInt64)(32 + frac) << (12 + dictMain)); + if (item.Is_Rar5_Compat()) + { + *s++ = ':'; + *s++ = 'c'; + } } } - unsigned cryptoSize = 0; - int cryptoOffset = item.FindExtra(NExtraID::kCrypto, cryptoSize); + const int cryptoOffset = item.FindExtra(NExtraID::kCrypto, cryptoSize); if (cryptoOffset >= 0) { - s = temp + strlen(temp); *s++ = ' '; - CCryptoInfo cryptoInfo; - - bool isOK = cryptoInfo.Parse(item.Extra + (unsigned)cryptoOffset, cryptoSize); - + const bool isOK = cryptoInfo.Parse(item.Extra + (unsigned)cryptoOffset, cryptoSize); if (cryptoInfo.Algo == 0) s = MyStpCpy(s, "AES"); else { s = MyStpCpy(s, "Crypto_"); - ConvertUInt64ToString(cryptoInfo.Algo, s); - s += strlen(s); + s = ConvertUInt64ToString(cryptoInfo.Algo, s); } - if (isOK) { *s++ = ':'; - ConvertUInt32ToString(cryptoInfo.Cnt, s); - s += strlen(s); + s = ConvertUInt32ToString(cryptoInfo.Cnt, s); *s++ = ':'; - ConvertUInt64ToString(cryptoInfo.Flags, s); + s = ConvertUInt64ToString(cryptoInfo.Flags, s); } } - + *s = 0; prop = temp; break; } @@ -1844,20 +2037,14 @@ Z7_COM7F_IMF(CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val AString s; if (item.ACL >= 0) - { s.Add_OptSpaced("ACL"); - } - - UInt32 flags = item.Flags; - // flags &= ~(6); // we don't need compression related bits here. + const UInt32 flags = item.Flags; if (flags != 0) { - AString s2 = FlagsToString(k_FileFlags, Z7_ARRAY_SIZE(k_FileFlags), flags); + const AString s2 = FlagsToString(k_FileFlags, Z7_ARRAY_SIZE(k_FileFlags), flags); if (!s2.IsEmpty()) - { s.Add_OptSpaced(s2); - } } item.PrintInfo(s); @@ -1895,7 +2082,7 @@ static int CompareItemsPaths(const CHandler &handler, unsigned p1, unsigned p2, { if (!item2.Version_Defined) return -1; - int res = MyCompare(item1.Version, item2.Version); + const int res = MyCompare(item1.Version, item2.Version); if (res != 0) return res; } @@ -1909,7 +2096,7 @@ static int CompareItemsPaths(const CHandler &handler, unsigned p1, unsigned p2, static int CompareItemsPaths2(const CHandler &handler, unsigned p1, unsigned p2, const AString *name1) { - int res = CompareItemsPaths(handler, p1, p2, name1); + const int res = CompareItemsPaths(handler, p1, p2, name1); if (res != 0) return res; return MyCompare(p1, p2); @@ -1958,15 +2145,34 @@ static int FindLink(const CHandler &handler, const CUIntVector &sorted, void CHandler::FillLinks() { unsigned i; - + + bool need_FillLinks = false; + for (i = 0; i < _refs.Size(); i++) { const CItem &item = _items[_refs[i].Item]; - if (!item.IsDir() && !item.IsService() && item.NeedUse_as_CopyLink()) - break; + if (!item.IsDir() + && !item.IsService() + && item.NeedUse_as_CopyLink()) + need_FillLinks = true; + + if (!item.IsSolid()) + _numBlocks++; + + const unsigned algo = item.Get_AlgoVersion_RawBits(); + _algo_Mask |= (UInt64)1 << algo; + _rar5comapt_mask |= 1u << item.Get_Rar5_CompatBit(); + if (!item.IsDir() && algo < Z7_ARRAY_SIZE(_methodMasks)) + { + _methodMasks[algo] |= 1u << (item.Get_Method()); + UInt64 d = 32 + item.Get_DictSize_Frac(); + d <<= (12 + item.Get_DictSize_Main()); + if (_dictMaxSizes[algo] < d) + _dictMaxSizes[algo] = d; + } } - if (i == _refs.Size()) + if (!need_FillLinks) return; CUIntVector sorted; @@ -1994,7 +2200,7 @@ void CHandler::FillLinks() if (!item.FindExtra_Link(linkInfo) || linkInfo.Type != NLinkType::kFileCopy) continue; link.SetFrom_CalcLen((const char *)(item.Extra + linkInfo.NameOffset), linkInfo.NameLen); - int linkIndex = FindLink(*this, sorted, link, i); + const int linkIndex = FindLink(*this, sorted, link, i); if (linkIndex < 0) continue; if ((unsigned)linkIndex >= i) @@ -2018,30 +2224,24 @@ HRESULT CHandler::Open2(IInStream *stream, IArchiveOpenCallback *openCallback) { CMyComPtr openVolumeCallback; - CMyComPtr getTextPassword; - + // CMyComPtr getTextPassword; NRar::CVolumeName seqName; - - UInt64 totalBytes = 0; - UInt64 curBytes = 0; + CTempBuf tempBuf; + CUnpacker unpacker; if (openCallback) { openCallback->QueryInterface(IID_IArchiveOpenVolumeCallback, (void **)&openVolumeCallback); - openCallback->QueryInterface(IID_ICryptoGetTextPassword, (void **)&getTextPassword); + openCallback->QueryInterface(IID_ICryptoGetTextPassword, (void **)&unpacker.getTextPassword); } + // unpacker.getTextPassword = getTextPassword; - CTempBuf tempBuf; - - CUnpacker unpacker; - unpacker.getTextPassword = getTextPassword; - + CInArchive arch; int prevSplitFile = -1; int prevMainFile = -1; - + UInt64 totalBytes = 0; + UInt64 curBytes = 0; bool nextVol_is_Required = false; - - CInArchive arch; for (;;) { @@ -2053,7 +2253,6 @@ HRESULT CHandler::Open2(IInStream *stream, { if (!openVolumeCallback) break; - if (_arcs.Size() == 1) { UString baseName; @@ -2067,14 +2266,10 @@ HRESULT CHandler::Open2(IInStream *stream, if (!seqName.InitName(baseName)) break; } - const UString volName = seqName.GetNextName(); - - HRESULT result = openVolumeCallback->GetStream(volName, &inStream); - + const HRESULT result = openVolumeCallback->GetStream(volName, &inStream); if (result != S_OK && result != S_FALSE) return result; - if (!inStream || result != S_OK) { if (nextVol_is_Required) @@ -2092,29 +2287,26 @@ HRESULT CHandler::Open2(IInStream *stream, RINOK(openCallback->SetTotal(NULL, &totalBytes)) } - CInArcInfo arcInfoOpen; + CInArcInfo arcInfo_Open; { - HRESULT res = arch.Open(inStream, maxCheckStartPosition, getTextPassword, arcInfoOpen); - if (arch.IsArc && arch.UnexpectedEnd) - _errorFlags |= kpv_ErrorFlags_UnexpectedEnd; - if (_arcs.IsEmpty()) - { - _isArc = arch.IsArc; - } - - if (res != S_OK) - { - if (res != S_FALSE) - return res; + const HRESULT res = arch.Open(inStream, maxCheckStartPosition, unpacker.getTextPassword, arcInfo_Open); + if (arch.IsArc && arch.UnexpectedEnd) + _errorFlags |= kpv_ErrorFlags_UnexpectedEnd; if (_arcs.IsEmpty()) - return res; - break; - } + _isArc = arch.IsArc; + if (res != S_OK) + { + if (res != S_FALSE) + return res; + if (_arcs.IsEmpty()) + return res; + break; + } } CArc &arc = _arcs.AddNew(); CInArcInfo &arcInfo = arc.Info; - arcInfo = arcInfoOpen; + arcInfo = arcInfo_Open; arc.Stream = inStream; CItem item; @@ -2124,18 +2316,16 @@ HRESULT CHandler::Open2(IInStream *stream, item.Clear(); arcInfo.EndPos = arch.Position; - if (arch.Position > endPos) { _errorFlags |= kpv_ErrorFlags_UnexpectedEnd; break; } - RINOK(InStream_SeekSet(inStream, arch.Position)) { CInArchive::CHeader h; - HRESULT res = arch.ReadBlockHeader(h); + const HRESULT res = arch.ReadBlockHeader(h); if (res != S_OK) { if (res != S_FALSE) @@ -2144,9 +2334,9 @@ HRESULT CHandler::Open2(IInStream *stream, { _errorFlags |= kpv_ErrorFlags_UnexpectedEnd; if (arcInfo.EndPos < arch.Position) - arcInfo.EndPos = arch.Position; + arcInfo.EndPos = arch.Position; if (arcInfo.EndPos < endPos) - arcInfo.EndPos = endPos; + arcInfo.EndPos = endPos; } else _errorFlags |= kpv_ErrorFlags_HeadersError; @@ -2159,6 +2349,8 @@ HRESULT CHandler::Open2(IInStream *stream, arcInfo.EndOfArchive_was_Read = true; if (!arch.ReadVar(arcInfo.EndFlags)) _errorFlags |= kpv_ErrorFlags_HeadersError; + if (!arch.Is_Buf_Finished() || h.ExtraSize || h.DataSize) + arcInfo.UnsupportedFeature = true; if (arcInfo.IsVolume()) { // for multivolume archives RAR can add ZERO bytes at the end for alignment. @@ -2182,14 +2374,11 @@ HRESULT CHandler::Open2(IInStream *stream, } item.RecordType = (Byte)h.Type; - if (!arch.ReadFileHeader(h, item)) { _errorFlags |= kpv_ErrorFlags_HeadersError; break; } - - // item.MainPartSize = (UInt32)(Position - item.Position); item.DataPos = arch.Position; } @@ -2201,7 +2390,7 @@ HRESULT CHandler::Open2(IInStream *stream, isOk_packSize = false; _errorFlags |= kpv_ErrorFlags_HeadersError; if (arcInfo.EndPos < endPos) - arcInfo.EndPos = endPos; + arcInfo.EndPos = endPos; } else { @@ -2211,29 +2400,32 @@ HRESULT CHandler::Open2(IInStream *stream, } bool needAdd = true; - + + if (!_comment_WasUsedInArc + && _comment.Size() == 0 + && item.Is_CMT()) { - if (_comment.Size() == 0 - && item.Is_CMT() - && item.PackSize < kCommentSize_Max + _comment_WasUsedInArc = true; + if ( item.PackSize <= kCommentSize_Max && item.PackSize == item.Size && item.PackSize != 0 - && item.GetMethod() == 0 + && item.Get_Method() == 0 && !item.IsSplit()) { RINOK(unpacker.DecodeToBuf(EXTERNAL_CODECS_VARS item, item.PackSize, inStream, _comment)) needAdd = false; + // item.RecordType = (Byte)NHeaderType::kFile; // for debug } } + + CRefItem ref; + ref.Item = _items.Size(); + ref.Last = ref.Item; + ref.Parent = -1; + ref.Link = -1; if (needAdd) { - CRefItem ref; - ref.Item = _items.Size(); - ref.Last = ref.Item; - ref.Parent = -1; - ref.Link = -1; - if (item.IsService()) { if (item.Is_STM()) @@ -2244,8 +2436,16 @@ HRESULT CHandler::Open2(IInStream *stream, else { needAdd = false; - if (item.Is_ACL() && (!item.IsEncrypted() || arch.m_CryptoMode)) + if (item.Is_ACL()) { + _acl_Used = true; + if (item.IsEncrypted() && !arch.m_CryptoMode) + _error_in_ACL = true; + else if (item.IsSolid() + || prevMainFile < 0 + || item.Size >= (1 << 24) + || item.Size == 0) + _error_in_ACL = true; if (prevMainFile >= 0 && item.Size < (1 << 24) && item.Size != 0) { CItem &mainItem = _items[_refs[prevMainFile].Item]; @@ -2253,7 +2453,7 @@ HRESULT CHandler::Open2(IInStream *stream, if (mainItem.ACL < 0) { CByteBuffer acl; - HRESULT res = tempBuf.Decode(EXTERNAL_CODECS_VARS item, inStream, unpacker, acl); + const HRESULT res = tempBuf.Decode(EXTERNAL_CODECS_VARS item, inStream, unpacker, acl); if (!item.IsSplitAfter()) tempBuf.Clear(); if (res != S_OK) @@ -2261,10 +2461,9 @@ HRESULT CHandler::Open2(IInStream *stream, tempBuf.Clear(); if (res != S_FALSE && res != E_NOTIMPL) return res; + _error_in_ACL = true; } - // RINOK(); - - if (res == S_OK && acl.Size() != 0) + else if (acl.Size() != 0) { if (_acls.IsEmpty() || acl != _acls.Back()) _acls.Add(acl); @@ -2274,7 +2473,7 @@ HRESULT CHandler::Open2(IInStream *stream, } } } - } + } // item.IsService() if (needAdd) { @@ -2291,6 +2490,8 @@ HRESULT CHandler::Open2(IInStream *stream, needAdd = false; } } + else + _split_Error = true; } } @@ -2300,7 +2501,6 @@ HRESULT CHandler::Open2(IInStream *stream, prevSplitFile = (int)_refs.Size(); if (!item.IsService()) prevMainFile = (int)_refs.Size(); - _refs.Add(ref); } } @@ -2315,11 +2515,13 @@ HRESULT CHandler::Open2(IInStream *stream, item.VolIndex = _arcs.Size() - 1; _items.Add(item); + if (needAdd) + _refs.Add(ref); if (openCallback && (_items.Size() & 0xFF) == 0) { - UInt64 numFiles = _items.Size(); - UInt64 numBytes = curBytes + item.DataPos; + const UInt64 numFiles = _refs.Size(); // _items.Size() + const UInt64 numBytes = curBytes + item.DataPos; RINOK(openCallback->SetCompleted(&numFiles, &numBytes)) } @@ -2343,11 +2545,11 @@ HRESULT CHandler::Open2(IInStream *stream, } FillLinks(); - return S_OK; } + Z7_COM7F_IMF(CHandler::Open(IInStream *stream, const UInt64 *maxCheckStartPosition, IArchiveOpenCallback *openCallback)) @@ -2365,6 +2567,19 @@ Z7_COM7F_IMF(CHandler::Close()) _errorFlags = 0; // _warningFlags = 0; _isArc = false; + _comment_WasUsedInArc = false; + _acl_Used = false; + _error_in_ACL = false; + _split_Error = false; + _numBlocks = 0; + _rar5comapt_mask = 0; + _algo_Mask = 0; // (UInt64)0u - 1; + for (unsigned i = 0; i < Z7_ARRAY_SIZE(_methodMasks); i++) + { + _methodMasks[i] = 0; + _dictMaxSizes[i] = 0; + } + _refs.Clear(); _items.Clear(); _arcs.Clear(); @@ -2427,7 +2642,7 @@ Z7_COM7F_IMF(CVolsInStream::Read(void *data, UInt32 size, UInt32 *processedSize) UInt32 cur = size; if (cur > _rem) cur = (UInt32)_rem; - UInt32 num = cur; + const UInt32 num = cur; HRESULT res = _stream->Read(data, cur, &cur); _hash.Update(data, cur); realProcessedSize += cur; @@ -2492,29 +2707,24 @@ static inline int DecoderRes_to_OpRes(HRESULT res, bool crcOK) static HRESULT CopyData_with_Progress(const Byte *data, size_t size, ISequentialOutStream *outStream, ICompressProgressInfo *progress) { - size_t pos = 0; - - while (pos < size) + UInt64 pos64 = 0; + while (size) { - const UInt32 kStepSize = ((UInt32)1 << 24); - UInt32 cur32; - { - size_t cur = size - pos; - if (cur > kStepSize) - cur = kStepSize; - cur32 = (UInt32)cur; - } - RINOK(outStream->Write(data + pos, cur32, &cur32)) - if (cur32 == 0) + const UInt32 kStepSize = (UInt32)1 << 24; + UInt32 cur = kStepSize; + if (cur > size) + cur = (UInt32)size; + RINOK(outStream->Write(data, cur, &cur)) + if (cur == 0) return E_FAIL; - pos += cur32; + size -= cur; + data += cur; + pos64 += cur; if (progress) { - UInt64 pos64 = pos; RINOK(progress->SetRatioInfo(&pos64, &pos64)) } } - return S_OK; } @@ -2525,7 +2735,7 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, COM_TRY_BEGIN const bool allFilesMode = (numItems == (UInt32)(Int32)-1); if (allFilesMode) - numItems = _refs.Size(); + numItems = (UInt32)_refs.Size(); if (numItems == 0) return S_OK; @@ -2554,12 +2764,11 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, UInt64 total = 0; bool isThereUndefinedSize = false; bool thereAreLinks = false; - { unsigned solidLimit = 0; for (UInt32 t = 0; t < numItems; t++) { - unsigned index = allFilesMode ? t : indices[t]; + const unsigned index = (unsigned)(allFilesMode ? t : indices[t]); const CRefItem &ref = _refs[index]; const CItem &item = _items[ref.Item]; const CItem &lastItem = _items[ref.Last]; @@ -2688,10 +2897,10 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, if (!testMode) for (UInt32 t = 0; t < numItems; t++) { - unsigned index = allFilesMode ? t : indices[t]; + const unsigned index = (unsigned)(allFilesMode ? t : indices[t]); const CRefItem &ref = _refs[index]; - int linkIndex = ref.Link; + const int linkIndex = ref.Link; if (linkIndex < 0 || (unsigned)linkIndex >= index) continue; const CItem &linkItem = _items[_refs[(unsigned)linkIndex].Item]; @@ -2711,38 +2920,139 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, } + + // ---------- MEMORY REQUEST ---------- + { + UInt64 dictMaxSize = 0; + for (UInt32 i = 0; i < _refs.Size(); i++) + { + if (extractStatuses[i] == 0) + continue; + const CRefItem &ref = _refs[i]; + const CItem &item = _items[ref.Item]; +/* + if (!item.IsDir() && !item.IsService() && item.NeedUse_as_CopyLink()) + { + } +*/ + const unsigned algo = item.Get_AlgoVersion_RawBits(); + if (!item.IsDir() && algo < Z7_ARRAY_SIZE(_methodMasks)) + { + const UInt64 d = item.Get_DictSize64(); + if (dictMaxSize < d) + dictMaxSize = d; + } + } + // we use callback, if dict exceeds (1 GB), because + // client code can set low limit (1 GB) for allowed memory usage. + const UInt64 k_MemLimit_for_Callback = (UInt64)1 << 30; + if (dictMaxSize > (_memUsage_WasSet ? + _memUsage_Decompress : k_MemLimit_for_Callback)) + { + { + CMyComPtr requestMem; + extractCallback->QueryInterface(IID_IArchiveRequestMemoryUseCallback, (void **)&requestMem); + if (!requestMem) + { + if (_memUsage_WasSet) + return E_OUTOFMEMORY; + } + else + { + UInt64 allowedSize = _memUsage_WasSet ? + _memUsage_Decompress : + (UInt64)1 << 32; // 4 GB is default allowed limit for RAR7 + + const UInt32 flags = (_memUsage_WasSet ? + NRequestMemoryUseFlags::k_AllowedSize_WasForced | + NRequestMemoryUseFlags::k_MLimit_Exceeded : + (dictMaxSize > allowedSize) ? + NRequestMemoryUseFlags::k_DefaultLimit_Exceeded: + 0) + | NRequestMemoryUseFlags::k_SkipArc_IsExpected + // | NRequestMemoryUseFlags::k_NoErrorMessage // for debug + ; + + // we set "Allow" for default case, if requestMem doesn't process anything. + UInt32 answerFlags = + (_memUsage_WasSet && dictMaxSize > allowedSize) ? + NRequestMemoryAnswerFlags::k_Limit_Exceeded + | NRequestMemoryAnswerFlags::k_SkipArc + : NRequestMemoryAnswerFlags::k_Allow; + + RINOK(requestMem->RequestMemoryUse( + flags, + NEventIndexType::kNoIndex, + // NEventIndexType::kInArcIndex, // for debug + 0, // index + NULL, // path + dictMaxSize, &allowedSize, &answerFlags)) + if ( (answerFlags & NRequestMemoryAnswerFlags::k_Allow) == 0 + || (answerFlags & NRequestMemoryAnswerFlags::k_Stop) + || (answerFlags & NRequestMemoryAnswerFlags::k_SkipArc) + ) + { + return E_OUTOFMEMORY; + } +/* + if ((answerFlags & NRequestMemoryAnswerFlags::k_AskForBigFile) == 0 && + (answerFlags & NRequestMemoryAnswerFlags::k_ReportForBigFile) == 0) + { + // requestMem.Release(); + } +*/ + } + } + } + } + + + + // ---------- UNPACK ---------- + UInt64 totalUnpacked = 0; UInt64 totalPacked = 0; - UInt64 curUnpackSize = 0; - UInt64 curPackSize = 0; + UInt64 curUnpackSize; + UInt64 curPackSize; CUnpacker unpacker; - - CVolsInStream *volsInStreamSpec = new CVolsInStream; - CMyComPtr volsInStream = volsInStreamSpec; - - CLocalProgress *lps = new CLocalProgress; - CMyComPtr progress = lps; + unpacker.NeedCrc = _needChecksumCheck; + CMyComPtr2_Create volsInStream; + CMyComPtr2_Create lps; lps->Init(extractCallback, false); - // bool needClearSolid = true; +/* + bool prevSolidWasSkipped = false; + UInt64 solidDictSize_Skip = 0; +*/ - FOR_VECTOR (i, _refs) + for (unsigned i = 0;; i++, + totalUnpacked += curUnpackSize, + totalPacked += curPackSize) { - if (extractStatuses[i] == 0) - continue; - - totalUnpacked += curUnpackSize; - totalPacked += curPackSize; lps->InSize = totalPacked; lps->OutSize = totalUnpacked; RINOK(lps->SetCur()) + { + const unsigned num = _refs.Size(); + if (i >= num) + break; + for (;;) + { + if (extractStatuses[i] != 0) + break; + i++; + if (i >= num) + break; + } + if (i >= num) + break; + } + curUnpackSize = 0; + curPackSize = 0; - CMyComPtr realOutStream; - // isExtract means that we don't skip that item. So we need read data. - - bool isExtract = ((extractStatuses[i] & kStatus_Extract) != 0); + const bool isExtract = ((extractStatuses[i] & kStatus_Extract) != 0); Int32 askMode = isExtract ? (testMode ? NExtract::NAskMode::kTest : @@ -2754,14 +3064,13 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, // if (!testMode) if ((extractStatuses[i] & kStatus_Link) != 0) { - int bufIndex = FindLinkBuf(linkFiles, i); + const int bufIndex = FindLinkBuf(linkFiles, i); if (bufIndex < 0) return E_FAIL; unpacker.linkFile = &linkFiles[bufIndex]; } - UInt32 index = i; - + const unsigned index = i; const CRefItem *ref = &_refs[index]; const CItem *item = &_items[ref->Item]; const CItem &lastItem = _items[ref->Last]; @@ -2772,8 +3081,6 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, curPackSize = GetPackSize(index); - RINOK(extractCallback->GetStream(index, &realOutStream, askMode)) - bool isSolid = false; if (!item->IsService()) { @@ -2782,6 +3089,52 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, unpacker.SolidAllowed = isSolid; } + + // ----- request mem ----- +/* + // link files are complicated cases. (ref->Link >= 0) + // link file can refer to non-solid file that can have big dictionary + // link file can refer to solid files that requres buffer + if (!item->IsDir() && requestMem && ref->Link < 0) + { + bool needSkip = false; + if (isSolid) + needSkip = prevSolidWasSkipped; + else + { + // isSolid == false + const unsigned algo = item->Get_AlgoVersion_RawBits(); + // const unsigned m = item.Get_Method(); + if (algo < Z7_ARRAY_SIZE(_methodMasks)) + { + solidDictSize_Skip = item->Get_DictSize64(); + if (solidDictSize_Skip > allowedSize) + needSkip = true; + } + } + if (needSkip) + { + UInt32 answerFlags = 0; + UInt64 allowedSize_File = allowedSize; + RINOK(requestMem->RequestMemoryUse( + NRequestMemoryUseFlags::k_Limit_Exceeded | + NRequestMemoryUseFlags::k_IsReport, + NEventIndexType::kInArcIndex, + index, + NULL, // path + solidDictSize_Skip, &allowedSize_File, &answerFlags)) + if (!item->IsService()) + prevSolidWasSkipped = true; + continue; + } + } + if (!item->IsService() && item->IsDir()) + prevSolidWasSkipped = false; +*/ + + CMyComPtr realOutStream; + RINOK(extractCallback->GetStream((UInt32)index, &realOutStream, askMode)) + if (item->IsDir()) { RINOK(extractCallback->PrepareOperation(askMode)) @@ -2789,7 +3142,7 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, continue; } - int index2 = ref->Link; + const int index2 = ref->Link; int bufIndex = -1; @@ -2880,7 +3233,7 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, if (needCallback) if (realOutStream) { - RINOK(CopyData_with_Progress(linkFile.Data, linkFile.Data.Size(), realOutStream, progress)) + RINOK(CopyData_with_Progress(linkFile.Data, linkFile.Data.Size(), realOutStream, lps)) } if (--linkFile.NumLinks == 0) @@ -2899,7 +3252,7 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, if (item->NeedUse_as_CopyLink()) { - int opRes = realOutStream ? + const int opRes = realOutStream ? NExtract::NOperationResult::kUnsupportedMethod: NExtract::NOperationResult::kOK; realOutStream.Release(); @@ -2907,9 +3260,9 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, continue; } - volsInStreamSpec->Init(&_arcs, &_items, ref->Item); + volsInStream->Init(&_arcs, &_items, ref->Item); - UInt64 packSize = curPackSize; + const UInt64 packSize = curPackSize; if (item->IsEncrypted()) if (!unpacker.getTextPassword) @@ -2927,9 +3280,9 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, bool crcOK = true; if (result == S_OK) - result = unpacker.Code(*item, _items[ref->Last], packSize, volsInStream, realOutStream, progress, crcOK); + result = unpacker.Code(*item, _items[ref->Last], packSize, volsInStream, realOutStream, lps, crcOK); realOutStream.Release(); - if (!volsInStreamSpec->CrcIsOK) + if (!volsInStream->CrcIsOK) crcOK = false; int opRes = crcOK ? @@ -2950,17 +3303,71 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, } { - FOR_VECTOR (i, linkFiles) - if (linkFiles[i].NumLinks != 0) + FOR_VECTOR (k, linkFiles) + if (linkFiles[k].NumLinks != 0) return E_FAIL; } return S_OK; - COM_TRY_END } +CHandler::CHandler() +{ + InitDefaults(); +} + +void CHandler::InitDefaults() +{ + _needChecksumCheck = true; + _memUsage_WasSet = false; + _memUsage_Decompress = (UInt64)1 << 32; +} + +Z7_COM7F_IMF(CHandler::SetProperties(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps)) +{ + InitDefaults(); + + for (UInt32 i = 0; i < numProps; i++) + { + UString name = names[i]; + name.MakeLower_Ascii(); + if (name.IsEmpty()) + return E_INVALIDARG; + + const PROPVARIANT &prop = values[i]; + + if (name.IsPrefixedBy_Ascii_NoCase("mt")) + { + } + else if (name.IsPrefixedBy_Ascii_NoCase("memx")) + { + UInt64 memAvail; + if (!NWindows::NSystem::GetRamSize(memAvail)) + memAvail = (UInt64)(sizeof(size_t)) << 28; + UInt64 v; + if (!ParseSizeString(name.Ptr(4), prop, memAvail, v)) + return E_INVALIDARG; + _memUsage_Decompress = v; + _memUsage_WasSet = true; + } + else if (name.IsPrefixedBy_Ascii_NoCase("crc")) + { + name.Delete(0, 3); + UInt32 crcSize = 1; + RINOK(ParsePropToUInt32(name, prop, crcSize)) + _needChecksumCheck = (crcSize != 0); + } + else + { + return E_INVALIDARG; + } + } + return S_OK; +} + + IMPL_ISetCompressCodecsInfo REGISTER_ARC_I( @@ -2973,29 +3380,78 @@ REGISTER_ARC_I( }} -Z7_CLASS_IMP_COM_1( +Z7_CLASS_IMP_COM_2( CBlake2spHasher , IHasher + , ICompressSetCoderProperties ) - CBlake2sp _blake; + CAlignedBuffer1 _buf; + // CBlake2sp _blake; + #define Z7_BLACK2S_ALIGN_OBJECT_OFFSET 0 + CBlake2sp *Obj() { return (CBlake2sp *)(void *)((Byte *)_buf + Z7_BLACK2S_ALIGN_OBJECT_OFFSET); } public: Byte _mtDummy[1 << 7]; // it's public to eliminate clang warning: unused private field - CBlake2spHasher() { Init(); } + CBlake2spHasher(): + _buf(sizeof(CBlake2sp) + Z7_BLACK2S_ALIGN_OBJECT_OFFSET) + { + Blake2sp_SetFunction(Obj(), 0); + Blake2sp_InitState(Obj()); + } }; Z7_COM7F_IMF2(void, CBlake2spHasher::Init()) { - Blake2sp_Init(&_blake); + Blake2sp_InitState(Obj()); } Z7_COM7F_IMF2(void, CBlake2spHasher::Update(const void *data, UInt32 size)) { - Blake2sp_Update(&_blake, (const Byte *)data, size); +#if 1 + Blake2sp_Update(Obj(), (const Byte *)data, (size_t)size); +#else + // for debug: + for (;;) + { + if (size == 0) + return; + UInt32 size2 = (size * 0x85EBCA87) % size / 800; + // UInt32 size2 = size / 2; + if (size2 == 0) + size2 = 1; + Blake2sp_Update(Obj(), (const Byte *)data, size2); + data = (const void *)((const Byte *)data + size2); + size -= size2; + } +#endif } Z7_COM7F_IMF2(void, CBlake2spHasher::Final(Byte *digest)) { - Blake2sp_Final(&_blake, digest); + Blake2sp_Final(Obj(), digest); } -REGISTER_HASHER(CBlake2spHasher, 0x202, "BLAKE2sp", BLAKE2S_DIGEST_SIZE) +Z7_COM7F_IMF(CBlake2spHasher::SetCoderProperties(const PROPID *propIDs, const PROPVARIANT *coderProps, UInt32 numProps)) +{ + unsigned algo = 0; + for (UInt32 i = 0; i < numProps; i++) + { + if (propIDs[i] == NCoderPropID::kDefaultProp) + { + const PROPVARIANT &prop = coderProps[i]; + if (prop.vt != VT_UI4) + return E_INVALIDARG; + /* + if (prop.ulVal > Z7_BLAKE2S_ALGO_MAX) + return E_NOTIMPL; + */ + algo = (unsigned)prop.ulVal; + } + } + if (!Blake2sp_SetFunction(Obj(), algo)) + return E_NOTIMPL; + return S_OK; +} + +REGISTER_HASHER(CBlake2spHasher, 0x202, "BLAKE2sp", Z7_BLAKE2S_DIGEST_SIZE) + +static struct CBlake2sp_Prepare { CBlake2sp_Prepare() { z7_Black2sp_Prepare(); } } g_Blake2sp_Prepare; diff --git a/CPP/7zip/Archive/Rar/Rar5Handler.h b/CPP/7zip/Archive/Rar/Rar5Handler.h index 72e1efc..8f6581a 100644 --- a/CPP/7zip/Archive/Rar/Rar5Handler.h +++ b/CPP/7zip/Archive/Rar/Rar5Handler.h @@ -48,7 +48,8 @@ namespace NArcFlags // const unsigned kLocked = 1 << 4; } -const unsigned kArcExtraRecordType_Locator = 1; +const unsigned kArcExtraRecordType_Locator = 1; +const unsigned kArcExtraRecordType_Metadata = 2; namespace NLocatorFlags { @@ -56,6 +57,14 @@ namespace NLocatorFlags const unsigned kRecovery = 1 << 1; } +namespace NMetadataFlags +{ + const unsigned kArcName = 1 << 0; + const unsigned kCTime = 1 << 1; + const unsigned kUnixTime = 1 << 2; + const unsigned kNanoSec = 1 << 3; +} + namespace NFileFlags { const unsigned kIsDir = 1 << 0; @@ -68,6 +77,7 @@ namespace NMethodFlags { // const unsigned kVersionMask = 0x3F; const unsigned kSolid = 1 << 6; + const unsigned kRar5_Compat = 1u << 20; } namespace NArcEndFlags @@ -181,7 +191,7 @@ struct CItem AString Name; unsigned VolIndex; - int NextItem; + int NextItem; // in _items{} UInt32 UnixMTime; UInt32 CRC; @@ -201,9 +211,12 @@ struct CItem void Clear() { - CommonFlags = 0; - Flags = 0; + // CommonFlags = 0; + // Flags = 0; + // UnixMTime = 0; + // CRC = 0; + VolIndex = 0; NextItem = -1; @@ -230,10 +243,46 @@ struct CItem // && false; } - bool IsSolid() const { return ((UInt32)Method & NMethodFlags::kSolid) != 0; } - unsigned GetAlgoVersion() const { return (unsigned)Method & 0x3F; } - unsigned GetMethod() const { return ((unsigned)Method >> 7) & 0x7; } - UInt32 GetDictSize() const { return (((UInt32)Method >> 10) & 0xF); } + // rar docs: Solid flag can be set only for file headers and is never set for service headers. + bool IsSolid() const { return ((UInt32)Method & NMethodFlags::kSolid) != 0; } + bool Is_Rar5_Compat() const { return ((UInt32)Method & NMethodFlags::kRar5_Compat) != 0; } + unsigned Get_Rar5_CompatBit() const { return ((UInt32)Method >> 20) & 1; } + + unsigned Get_AlgoVersion_RawBits() const { return (unsigned)Method & 0x3F; } + unsigned Get_AlgoVersion_HuffRev() const + { + unsigned w = (unsigned)Method & 0x3F; + if (w == 1 && Is_Rar5_Compat()) + w = 0; + return w; + } + unsigned Get_Method() const { return ((unsigned)Method >> 7) & 0x7; } + + unsigned Get_DictSize_Main() const + { return ((UInt32)Method >> 10) & (Get_AlgoVersion_RawBits() == 0 ? 0xf : 0x1f); } + unsigned Get_DictSize_Frac() const + { + // original-unrar ignores Frac, if (algo==0) (rar5): + if (Get_AlgoVersion_RawBits() == 0) + return 0; + return ((UInt32)Method >> 15) & 0x1f; + } + UInt64 Get_DictSize64() const + { + // ver 6.* check + // return (((UInt32)Method >> 10) & 0xF); + UInt64 winSize = 0; + const unsigned algo = Get_AlgoVersion_RawBits(); + if (algo <= 1) + { + UInt32 w = 32; + if (algo == 1) + w += Get_DictSize_Frac(); + winSize = (UInt64)w << (12 + Get_DictSize_Main()); + } + return winSize; + } + bool IsService() const { return RecordType == NHeaderType::kService; } @@ -255,9 +304,9 @@ struct CItem int FindExtra_Blake() const { unsigned size = 0; - int offset = FindExtra(NExtraID::kHash, size); + const int offset = FindExtra(NExtraID::kHash, size); if (offset >= 0 - && size == BLAKE2S_DIGEST_SIZE + 1 + && size == Z7_BLAKE2S_DIGEST_SIZE + 1 && Extra[(unsigned)offset] == kHashID_Blake2sp) return offset + 1; return -1; @@ -282,11 +331,17 @@ struct CItem UInt32 a; switch (HostOS) { - case kHost_Windows: a = Attrib; break; - case kHost_Unix: a = (Attrib << 16); break; - default: a = 0; + case kHost_Windows: + a = Attrib; + break; + case kHost_Unix: + a = Attrib << 16; + a |= 0x8000; // add posix mode marker + break; + default: + a = 0; } - // if (IsDir()) a |= FILE_ATTRIBUTE_DIRECTORY; + if (IsDir()) a |= FILE_ATTRIBUTE_DIRECTORY; return a; } @@ -305,10 +360,14 @@ struct CInArcInfo bool EndOfArchive_was_Read; bool IsEncrypted; + bool Locator_Defined; + bool Locator_Error; + bool Metadata_Defined; + bool Metadata_Error; + bool UnknownExtraRecord; + bool Extra_Error; + bool UnsupportedFeature; - // CByteBuffer Extra; - - /* struct CLocator { UInt64 Flags; @@ -317,11 +376,32 @@ struct CInArcInfo bool Is_QuickOpen() const { return (Flags & NLocatorFlags::kQuickOpen) != 0; } bool Is_Recovery() const { return (Flags & NLocatorFlags::kRecovery) != 0; } + + bool Parse(const Byte *p, size_t size); + CLocator(): + Flags(0), + QuickOpen(0), + Recovery(0) + {} }; - int FindExtra(unsigned extraID, unsigned &recordDataSize) const; - bool FindExtra_Locator(CLocator &locator) const; - */ + struct CMetadata + { + UInt64 Flags; + UInt64 CTime; + AString ArcName; + + bool Parse(const Byte *p, size_t size); + CMetadata(): + Flags(0), + CTime(0) + {} + }; + + CLocator Locator; + CMetadata Metadata; + + bool ParseExtra(const Byte *p, size_t size); CInArcInfo(): Flags(0), @@ -330,7 +410,14 @@ struct CInArcInfo EndPos(0), EndFlags(0), EndOfArchive_was_Read(false), - IsEncrypted(false) + IsEncrypted(false), + Locator_Defined(false), + Locator_Error(false), + Metadata_Defined(false), + Metadata_Error(false), + UnknownExtraRecord(false), + Extra_Error(false), + UnsupportedFeature(false) {} /* @@ -342,7 +429,6 @@ struct CInArcInfo EndPos = 0; EndFlags = 0; EndOfArchive_was_Read = false; - Extra.Free(); } */ @@ -360,10 +446,10 @@ struct CInArcInfo struct CRefItem { - unsigned Item; - unsigned Last; - int Parent; - int Link; + unsigned Item; // First item in _items[] + unsigned Last; // Last item in _items[] + int Parent; // in _refs[], if alternate stream + int Link; // in _refs[] }; @@ -377,32 +463,54 @@ struct CArc class CHandler Z7_final: public IInArchive, public IArchiveGetRawProps, + public ISetProperties, Z7_PUBLIC_ISetCompressCodecsInfo_IFEC public CMyUnknownImp { Z7_COM_QI_BEGIN2(IInArchive) Z7_COM_QI_ENTRY(IArchiveGetRawProps) + Z7_COM_QI_ENTRY(ISetProperties) Z7_COM_QI_ENTRY_ISetCompressCodecsInfo_IFEC Z7_COM_QI_END Z7_COM_ADDREF_RELEASE Z7_IFACE_COM7_IMP(IInArchive) Z7_IFACE_COM7_IMP(IArchiveGetRawProps) + Z7_IFACE_COM7_IMP(ISetProperties) DECL_ISetCompressCodecsInfo + void InitDefaults(); + + bool _isArc; + bool _needChecksumCheck; + bool _memUsage_WasSet; + bool _comment_WasUsedInArc; + bool _acl_Used; + bool _error_in_ACL; + bool _split_Error; public: CRecordVector _refs; CObjectVector _items; + + CHandler(); private: CObjectVector _arcs; CObjectVector _acls; UInt32 _errorFlags; // UInt32 _warningFlags; - bool _isArc; + + UInt32 _numBlocks; + unsigned _rar5comapt_mask; + unsigned _methodMasks[2]; + UInt64 _algo_Mask; + UInt64 _dictMaxSizes[2]; + CByteBuffer _comment; UString _missingVolName; + UInt64 _memUsage_Decompress; + DECL_EXTERNAL_CODECS_VARS UInt64 GetPackSize(unsigned refIndex) const; diff --git a/CPP/7zip/Archive/Rar/RarHandler.h b/CPP/7zip/Archive/Rar/RarHandler.h index c4d8450..6a1da4f 100644 --- a/CPP/7zip/Archive/Rar/RarHandler.h +++ b/CPP/7zip/Archive/Rar/RarHandler.h @@ -80,6 +80,8 @@ class CHandler Z7_final: Z7_IFACE_COM7_IMP(IInArchive) DECL_ISetCompressCodecsInfo + bool _isArc; + CRecordVector _refItems; CObjectVector _items; CObjectVector _arcs; @@ -87,7 +89,6 @@ class CHandler Z7_final: // AString _errorMessage; UInt32 _errorFlags; UInt32 _warningFlags; - bool _isArc; UString _missingVolName; DECL_EXTERNAL_CODECS_VARS @@ -99,7 +100,7 @@ class CHandler Z7_final: void AddErrorMessage(const AString &s) { if (!_errorMessage.IsEmpty()) - _errorMessage += '\n'; + _errorMessage.Add_LF(); _errorMessage += s; } */ diff --git a/CPP/7zip/Archive/RpmHandler.cpp b/CPP/7zip/Archive/RpmHandler.cpp index e05ac26..da2b6ee 100644 --- a/CPP/7zip/Archive/RpmHandler.cpp +++ b/CPP/7zip/Archive/RpmHandler.cpp @@ -97,7 +97,15 @@ static const char * const k_CPUs[] = , "ppc64" , "sh" , "xtensa" - , "aarch64" // 19 + , "aarch64" // 19 + , "mipsr6" // 20 + , "mips64r6" // 21 + , "riscv64" // 22 + , "loongarch64" // 23 + // , "24" + // , "25" + // , "loongarch64" // 26 : why 23 and 26 for loongarch64? + // 255 for some non specified arch }; static const char * const k_OS[] = @@ -128,8 +136,8 @@ static const char * const k_OS[] = struct CLead { - unsigned char Major; - unsigned char Minor; + Byte Major; + // Byte Minor; UInt16 Type; UInt16 Cpu; UInt16 Os; @@ -140,7 +148,7 @@ struct CLead void Parse(const Byte *p) { Major = p[4]; - Minor = p[5]; + // Minor = p[5]; Type = Get16(p + 6); Cpu= Get16(p + 8); memcpy(Name, p + 10, kNameSize); @@ -205,7 +213,7 @@ Z7_class_CHandler_final: public CHandlerCont AString _os; // linux AString _format; // cpio - AString _compressor; // xz, gzip, bzip2 + AString _compressor; // xz, gzip, bzip2, lzma, zstd CLead _lead; @@ -326,16 +334,20 @@ void CHandler::AddSubFileExtension(AString &res) const s = "bz2"; else if (_compressor == "gzip") s = "gz"; + else if (_compressor == "zstd") + s = "zst"; } else { const Byte *p = _payloadSig; - if (p[0] == 0x1F && p[1] == 0x8B) + if (p[0] == 0x1F && p[1] == 0x8B && p[2] == 8) s = "gz"; else if (p[0] == 0xFD && p[1] == '7' && p[2] == 'z' && p[3] == 'X' && p[4] == 'Z' && p[5] == 0) s = "xz"; else if (p[0] == 'B' && p[1] == 'Z' && p[2] == 'h' && p[3] >= '1' && p[3] <= '9') s = "bz2"; + else if (p[0] == 0x28 && p[1] == 0xb5 && p[2] == 0x2f && p[3] == 0xfd) + s = "zst"; else s = "lzma"; } @@ -466,12 +478,6 @@ Z7_COM7F_IMF(CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val return S_OK; } -#ifdef Z7_RPM_SHOW_METADATA -static inline char GetHex(unsigned value) -{ - return (char)((value < 10) ? ('0' + value) : ('A' + (value - 10))); -} -#endif HRESULT CHandler::ReadHeader(ISequentialInStream *stream, bool isMainHeader) { @@ -585,7 +591,7 @@ HRESULT CHandler::ReadHeader(ISequentialInStream *stream, bool isMainHeader) if (rem2 == 0) return S_FALSE; if (t != 0) - _metadata += '\n'; + _metadata.Add_LF(); size_t j; for (j = 0; j < rem2 && p2[j] != 0; j++); if (j == rem2) @@ -614,8 +620,8 @@ HRESULT CHandler::ReadHeader(ISequentialInStream *stream, bool isMainHeader) for (UInt32 t = 0; t < entry.Count; t++) { const unsigned b = p[t]; - _metadata += GetHex((b >> 4) & 0xF); - _metadata += GetHex(b & 0xF); + _metadata += GET_HEX_CHAR_UPPER(b >> 4); + _metadata += GET_HEX_CHAR_UPPER(b & 0xF); } } else @@ -623,7 +629,7 @@ HRESULT CHandler::ReadHeader(ISequentialInStream *stream, bool isMainHeader) // p = p; } - _metadata += '\n'; + _metadata.Add_LF(); #endif } diff --git a/CPP/7zip/Archive/SparseHandler.cpp b/CPP/7zip/Archive/SparseHandler.cpp index ab76f32..5e666ed 100644 --- a/CPP/7zip/Archive/SparseHandler.cpp +++ b/CPP/7zip/Archive/SparseHandler.cpp @@ -77,6 +77,15 @@ struct CChunk UInt32 VirtBlock; Byte Fill [kFillSize]; UInt64 PhyOffset; + + CChunk() + { + Fill[0] = + Fill[1] = + Fill[2] = + Fill[3] = + 0; + } }; static const Byte k_Signature[] = { 0x3a, 0xff, 0x26, 0xed, 1, 0 }; diff --git a/CPP/7zip/Archive/SplitHandler.cpp b/CPP/7zip/Archive/SplitHandler.cpp index ae7ca03..bf89303 100644 --- a/CPP/7zip/Archive/SplitHandler.cpp +++ b/CPP/7zip/Archive/SplitHandler.cpp @@ -56,6 +56,7 @@ Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) case kpidPhySize: if (!_sizes.IsEmpty()) prop = _sizes[0]; break; case kpidTotalPhySize: prop = _totalSize; break; case kpidNumVolumes: prop = (UInt32)_streams.Size(); break; + default: break; } prop.Detach(value); return S_OK; @@ -273,6 +274,7 @@ Z7_COM7F_IMF(CHandler::GetProperty(UInt32 /* index */, PROPID propID, PROPVARIAN case kpidPackSize: prop = _totalSize; break; + default: break; } prop.Detach(value); return S_OK; diff --git a/CPP/7zip/Archive/SquashfsHandler.cpp b/CPP/7zip/Archive/SquashfsHandler.cpp index 9c9ec03..46e0e1e 100644 --- a/CPP/7zip/Archive/SquashfsHandler.cpp +++ b/CPP/7zip/Archive/SquashfsHandler.cpp @@ -3,9 +3,10 @@ #include "StdAfx.h" #include "../../../C/Alloc.h" -#include "../../../C/CpuArch.h" #include "../../../C/LzmaDec.h" #include "../../../C/Xz.h" +#include "../../../C/ZstdDec.h" +#include "../../../C/CpuArch.h" #include "../../Common/ComTry.h" #include "../../Common/MyLinux.h" @@ -30,8 +31,8 @@ namespace NArchive { namespace NSquashfs { -static const UInt32 kNumFilesMax = (1 << 28); -static const unsigned kNumDirLevelsMax = (1 << 10); +static const UInt32 kNumFilesMax = 1 << 28; +static const unsigned kNumDirLevelsMax = 1 << 10; // Layout: Header, Data, inodes, Directories, Fragments, UIDs, GIDs @@ -66,6 +67,8 @@ static const UInt32 kSignature32_B2 = 0x73687371; #define kMethod_LZMA 2 #define kMethod_LZO 3 #define kMethod_XZ 4 +// #define kMethod_LZ4 5 +#define kMethod_ZSTD 6 static const char * const k_Methods[] = { @@ -74,6 +77,8 @@ static const char * const k_Methods[] = , "LZMA" , "LZO" , "XZ" + , "LZ4" + , "ZSTD" }; static const unsigned kMetadataBlockSizeLog = 13; @@ -127,8 +132,8 @@ static const char * const k_Flags[] = , "UNCOMPRESSED_IDS" }; -static const UInt32 kNotCompressedBit16 = (1 << 15); -static const UInt32 kNotCompressedBit32 = (1 << 24); +static const UInt32 kNotCompressedBit16 = 1 << 15; +static const UInt32 kNotCompressedBit32 = 1 << 24; #define GET_COMPRESSED_BLOCK_SIZE(size) ((size) & ~kNotCompressedBit32) #define IS_COMPRESSED_BLOCK(size) (((size) & kNotCompressedBit32) == 0) @@ -835,6 +840,9 @@ struct CFrag Z7_CLASS_IMP_CHandler_IInArchive_1( IInArchiveGetStream ) + bool _noPropsLZMA; + bool _needCheckLzma; + CRecordVector _items; CRecordVector _nodes; CRecordVector _nodesPos; @@ -845,16 +853,13 @@ Z7_CLASS_IMP_CHandler_IInArchive_1( CByteBuffer _uids; CByteBuffer _gids; CHeader _h; - bool _noPropsLZMA; - bool _needCheckLzma; - UInt32 _openCodePage; - - CMyComPtr _stream; UInt64 _sizeCalculated; + CMyComPtr _stream; IArchiveOpenCallback *_openCallback; + UInt32 _openCodePage; int _nodeIndex; CRecordVector _blockCompressed; CRecordVector _blockOffsets; @@ -864,25 +869,18 @@ Z7_CLASS_IMP_CHandler_IInArchive_1( UInt32 _cachedPackBlockSize; UInt32 _cachedUnpackBlockSize; - CLimitedSequentialInStream *_limitedInStreamSpec; - CMyComPtr _limitedInStream; - - CBufPtrSeqOutStream *_outStreamSpec; - CMyComPtr _outStream; - - // NCompress::NLzma::CDecoder *_lzmaDecoderSpec; - // CMyComPtr _lzmaDecoder; + CMyComPtr2_Create _limitedInStream; + CMyComPtr2_Create _outStream; + CMyComPtr2_Create _dynOutStream; - NCompress::NZlib::CDecoder *_zlibDecoderSpec; - CMyComPtr _zlibDecoder; + // CMyComPtr2 _lzmaDecoder; + CMyComPtr2 _zlibDecoder; CXzUnpacker _xz; + CZstdDecHandle _zstd; CByteBuffer _inputBuffer; - CDynBufSeqOutStream *_dynOutStreamSpec; - CMyComPtr _dynOutStream; - void ClearCache() { _cachedBlockStartPos = 0; @@ -913,24 +911,18 @@ Z7_CLASS_IMP_CHandler_IInArchive_1( ~CHandler() { XzUnpacker_Free(&_xz); + if (_zstd) + ZstdDec_Destroy(_zstd); } HRESULT ReadBlock(UInt64 blockIndex, Byte *dest, size_t blockSize); }; -CHandler::CHandler() +CHandler::CHandler(): + _zstd(NULL) { XzUnpacker_Construct(&_xz, &g_Alloc); - - _limitedInStreamSpec = new CLimitedSequentialInStream; - _limitedInStream = _limitedInStreamSpec; - - _outStreamSpec = new CBufPtrSeqOutStream(); - _outStream = _outStreamSpec; - - _dynOutStreamSpec = new CDynBufSeqOutStream; - _dynOutStream = _dynOutStreamSpec; } static const Byte kProps[] = @@ -1153,24 +1145,16 @@ HRESULT CHandler::Decompress(ISequentialOutStream *outStream, Byte *outBuf, bool if (method == kMethod_ZLIB) { - if (!_zlibDecoder) - { - _zlibDecoderSpec = new NCompress::NZlib::CDecoder(); - _zlibDecoder = _zlibDecoderSpec; - } - RINOK(_zlibDecoder->Code(_limitedInStream, outStream, NULL, NULL, NULL)) - if (inSize != _zlibDecoderSpec->GetInputProcessedSize()) + _zlibDecoder.Create_if_Empty(); + RINOK(_zlibDecoder.Interface()->Code(_limitedInStream, outStream, NULL, NULL, NULL)) + if (inSize != _zlibDecoder->GetInputProcessedSize()) return S_FALSE; } /* else if (method == kMethod_LZMA) { - if (!_lzmaDecoder) - { - _lzmaDecoderSpec = new NCompress::NLzma::CDecoder(); - // _lzmaDecoderSpec->FinishStream = true; - _lzmaDecoder = _lzmaDecoderSpec; - } + _lzmaDecoder.Create_if_Empty(); + // _lzmaDecoder->FinishStream = true; const UInt32 kPropsSize = LZMA_PROPS_SIZE + 8; Byte props[kPropsSize]; UInt32 propsSize; @@ -1205,7 +1189,7 @@ HRESULT CHandler::Decompress(ISequentialOutStream *outStream, Byte *outBuf, bool Byte *dest = outBuf; if (!outBuf) { - dest = _dynOutStreamSpec->GetBufPtrForWriting(outSizeMax); + dest = _dynOutStream->GetBufPtrForWriting(outSizeMax); if (!dest) return E_OUTOFMEMORY; } @@ -1253,10 +1237,67 @@ HRESULT CHandler::Decompress(ISequentialOutStream *outStream, Byte *outBuf, bool && status != LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK) return S_FALSE; } + else if (method == kMethod_ZSTD) + { + const Byte *src = _inputBuffer; + + if (!_zstd) + { + _zstd = ZstdDec_Create(&g_AlignedAlloc, &g_AlignedAlloc); + if (!_zstd) + return E_OUTOFMEMORY; + } + + CZstdDecState state; + ZstdDecState_Clear(&state); + + state.inBuf = src; + state.inLim = srcLen; // + 1; for debug + // state.outStep = outSizeMax; + + state.outBuf_fromCaller = dest; + state.outBufSize_fromCaller = outSizeMax; + // state.mustBeFinished = True; + + ZstdDec_Init(_zstd); + SRes sres; + for (;;) + { + sres = ZstdDec_Decode(_zstd, &state); + if (sres != SZ_OK) + break; + if (state.inLim == state.inPos + && (state.status == ZSTD_STATUS_NEEDS_MORE_INPUT || + state.status == ZSTD_STATUS_FINISHED_FRAME)) + break; + // sres = sres; + // break; // for debug + } + + CZstdDecResInfo info; + // ZstdDecInfo_Clear(&stat); + // stat->InSize = state.inPos; + ZstdDec_GetResInfo(_zstd, &state, sres, &info); + sres = info.decode_SRes; + if (sres == SZ_OK) + { + if (state.status != ZSTD_STATUS_FINISHED_FRAME + // ||stat.UnexpededEnd + || info.extraSize != 0 + || state.inLim != state.inPos) + sres = SZ_ERROR_DATA; + } + if (sres != SZ_OK) + return SResToHRESULT(sres); + if (state.winPos > outSizeMax) + return E_FAIL; + // memcpy(dest, state.dic, state.dicPos); + destLen = state.winPos; + } else { ECoderStatus status; - SRes res = XzUnpacker_CodeFull(&_xz, + const SRes res = XzUnpacker_CodeFull(&_xz, dest, &destLen, _inputBuffer, &srcLen, CODER_FINISH_END, &status); @@ -1274,7 +1315,7 @@ HRESULT CHandler::Decompress(ISequentialOutStream *outStream, Byte *outBuf, bool *outBufWasWrittenSize = (UInt32)destLen; } else - _dynOutStreamSpec->UpdateSize(destLen); + _dynOutStream->UpdateSize(destLen); } return S_OK; } @@ -1298,17 +1339,17 @@ HRESULT CHandler::ReadMetadataBlock(UInt32 &packSize) packSize = offset + size; if (isCompressed) { - _limitedInStreamSpec->Init(size); + _limitedInStream->Init(size); RINOK(Decompress(_dynOutStream, NULL, NULL, NULL, size, kMetadataBlockSize)) } else { // size != 0 here - Byte *buf = _dynOutStreamSpec->GetBufPtrForWriting(size); + Byte *buf = _dynOutStream->GetBufPtrForWriting(size); if (!buf) return E_OUTOFMEMORY; RINOK(ReadStream_FALSE(_stream, buf, size)) - _dynOutStreamSpec->UpdateSize(size); + _dynOutStream->UpdateSize(size); } return S_OK; } @@ -1316,7 +1357,7 @@ HRESULT CHandler::ReadMetadataBlock(UInt32 &packSize) HRESULT CHandler::ReadMetadataBlock2() { - _dynOutStreamSpec->Init(); + _dynOutStream->Init(); UInt32 packSize = kMetadataBlockSize + 3; // check it return ReadMetadataBlock(packSize); } @@ -1327,25 +1368,25 @@ HRESULT CHandler::ReadData(CData &data, UInt64 start, UInt64 end) return S_FALSE; const UInt32 size = (UInt32)(end - start); RINOK(Seek2(start)) - _dynOutStreamSpec->Init(); + _dynOutStream->Init(); UInt32 packPos = 0; while (packPos != size) { data.PackPos.Add(packPos); - data.UnpackPos.Add((UInt32)_dynOutStreamSpec->GetSize()); + data.UnpackPos.Add((UInt32)_dynOutStream->GetSize()); if (packPos > size) return S_FALSE; UInt32 packSize = size - packPos; RINOK(ReadMetadataBlock(packSize)) { - const size_t tSize = _dynOutStreamSpec->GetSize(); + const size_t tSize = _dynOutStream->GetSize(); if (tSize != (UInt32)tSize) return S_FALSE; } packPos += packSize; } - data.UnpackPos.Add((UInt32)_dynOutStreamSpec->GetSize()); - _dynOutStreamSpec->CopyToBuffer(data.Data); + data.UnpackPos.Add((UInt32)_dynOutStream->GetSize()); + _dynOutStream->CopyToBuffer(data.Data); return S_OK; } @@ -1455,7 +1496,7 @@ HRESULT CHandler::OpenDir(int parent, UInt32 startBlock, UInt32 offset, unsigned if (rem < nameOffset) return S_FALSE; - if ((UInt32)_items.Size() >= kNumFilesMax) + if (_items.Size() >= kNumFilesMax) return S_FALSE; if (_openCallback) { @@ -1557,6 +1598,7 @@ HRESULT CHandler::Open2(IInStream *inStream) case kMethod_LZMA: case kMethod_LZO: case kMethod_XZ: + case kMethod_ZSTD: break; default: return E_NOTIMPL; @@ -1585,11 +1627,11 @@ HRESULT CHandler::Open2(IInStream *inStream) const UInt64 offset = bigFrag ? Get64(data + i * 8) : Get32(data + i * 4); RINOK(Seek2(offset)) RINOK(ReadMetadataBlock2()) - const UInt32 unpackSize = (UInt32)_dynOutStreamSpec->GetSize(); + const UInt32 unpackSize = (UInt32)_dynOutStream->GetSize(); if (unpackSize != kMetadataBlockSize) if (i != numBlocks - 1 || unpackSize != ((_h.NumFrags << (3 + bigFrag)) & (kMetadataBlockSize - 1))) return S_FALSE; - const Byte *buf = _dynOutStreamSpec->GetBuffer(); + const Byte *buf = _dynOutStream->GetBuffer(); for (UInt32 j = 0; j < kMetadataBlockSize && j < unpackSize;) { CFrag frag; @@ -1683,12 +1725,12 @@ HRESULT CHandler::Open2(IInStream *inStream) RINOK(Seek2(offset)) // RINOK(ReadMetadataBlock(NULL, _uids + kMetadataBlockSize * i, packSize, unpackSize)); RINOK(ReadMetadataBlock2()) - const size_t unpackSize = _dynOutStreamSpec->GetSize(); + const size_t unpackSize = _dynOutStream->GetSize(); const UInt32 remSize = (i == numBlocks - 1) ? (size & (kMetadataBlockSize - 1)) : kMetadataBlockSize; if (unpackSize != remSize) return S_FALSE; - memcpy(_uids + kMetadataBlockSize * i, _dynOutStreamSpec->GetBuffer(), remSize); + memcpy(_uids + kMetadataBlockSize * i, _dynOutStream->GetBuffer(), remSize); } } @@ -1758,7 +1800,7 @@ Z7_COM7F_IMF(CHandler::Open(IInStream *stream, const UInt64 *, IArchiveOpenCallb COM_TRY_BEGIN { Close(); - _limitedInStreamSpec->SetStream(stream); + _limitedInStream->SetStream(stream); HRESULT res; try { @@ -1786,7 +1828,7 @@ Z7_COM7F_IMF(CHandler::Close()) _openCodePage = CP_UTF8; _sizeCalculated = 0; - _limitedInStreamSpec->ReleaseStream(); + _limitedInStream->ReleaseStream(); _stream.Release(); _items.Clear(); @@ -2126,11 +2168,11 @@ HRESULT CHandler::ReadBlock(UInt64 blockIndex, Byte *dest, size_t blockSize) { ClearCache(); RINOK(Seek2(blockOffset)) - _limitedInStreamSpec->Init(packBlockSize); + _limitedInStream->Init(packBlockSize); if (compressed) { - _outStreamSpec->Init((Byte *)_cachedBlock, _h.BlockSize); + _outStream->Init((Byte *)_cachedBlock, _h.BlockSize); bool outBufWasWritten; UInt32 outBufWasWrittenSize; HRESULT res = Decompress(_outStream, _cachedBlock, &outBufWasWritten, &outBufWasWrittenSize, packBlockSize, _h.BlockSize); @@ -2138,7 +2180,7 @@ HRESULT CHandler::ReadBlock(UInt64 blockIndex, Byte *dest, size_t blockSize) if (outBufWasWritten) _cachedUnpackBlockSize = outBufWasWrittenSize; else - _cachedUnpackBlockSize = (UInt32)_outStreamSpec->GetPos(); + _cachedUnpackBlockSize = (UInt32)_outStream->GetPos(); } else { @@ -2174,23 +2216,25 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, const CNode &node = _nodes[item.Node]; totalSize += node.GetSize(); } - extractCallback->SetTotal(totalSize); + RINOK(extractCallback->SetTotal(totalSize)) UInt64 totalPackSize; totalSize = totalPackSize = 0; - NCompress::CCopyCoder *copyCoderSpec = new NCompress::CCopyCoder(); - CMyComPtr copyCoder = copyCoderSpec; - - CLocalProgress *lps = new CLocalProgress; - CMyComPtr progress = lps; + CMyComPtr2_Create lps; lps->Init(extractCallback, false); + CMyComPtr2_Create copyCoder; - for (i = 0; i < numItems; i++) + for (i = 0;; i++) { lps->InSize = totalPackSize; lps->OutSize = totalSize; RINOK(lps->SetCur()) + if (i >= numItems) + break; + + int res; + { CMyComPtr outStream; const Int32 askMode = testMode ? NExtract::NAskMode::kTest : @@ -2207,7 +2251,7 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, RINOK(extractCallback->SetOperationResult(NExtract::NOperationResult::kOK)) continue; } - UInt64 unpackSize = node.GetSize(); + const UInt64 unpackSize = node.GetSize(); totalSize += unpackSize; UInt64 packSize; if (GetPackSize(index, packSize, false)) @@ -2217,7 +2261,7 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, continue; RINOK(extractCallback->PrepareOperation(askMode)) - int res = NExtract::NOperationResult::kDataError; + res = NExtract::NOperationResult::kDataError; { CMyComPtr inSeqStream; HRESULT hres = GetStream(index, &inSeqStream); @@ -2231,10 +2275,10 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, { RINOK(hres) { - hres = copyCoder->Code(inSeqStream, outStream, NULL, NULL, progress); + hres = copyCoder.Interface()->Code(inSeqStream, outStream, NULL, NULL, lps); if (hres == S_OK) { - if (copyCoderSpec->TotalSize == unpackSize) + if (copyCoder->TotalSize == unpackSize) res = NExtract::NOperationResult::kOK; } else if (hres == E_NOTIMPL) @@ -2248,7 +2292,7 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, } } } - + } RINOK(extractCallback->SetOperationResult(res)) } diff --git a/CPP/7zip/Archive/SwfHandler.cpp b/CPP/7zip/Archive/SwfHandler.cpp index ab704b6..b143d9a 100644 --- a/CPP/7zip/Archive/SwfHandler.cpp +++ b/CPP/7zip/Archive/SwfHandler.cpp @@ -179,10 +179,10 @@ struct CItem CMyComPtr _seqStream; CMyComPtr _stream; - #ifdef Z7_SWF_UPDATE +#ifdef Z7_SWF_UPDATE CSingleMethodProps _props; bool _lzmaMode; - #endif +#endif public: #ifdef Z7_SWF_UPDATE @@ -333,6 +333,8 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, return E_INVALIDARG; RINOK(extractCallback->SetTotal(_item.GetSize())) + Int32 opRes; +{ CMyComPtr realOutStream; const Int32 askMode = testMode ? NExtract::NAskMode::kTest : @@ -341,20 +343,18 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, if (!testMode && !realOutStream) return S_OK; - extractCallback->PrepareOperation(askMode); + RINOK(extractCallback->PrepareOperation(askMode)) - CDummyOutStream *outStreamSpec = new CDummyOutStream; - CMyComPtr outStream(outStreamSpec); - outStreamSpec->SetStream(realOutStream); - outStreamSpec->Init(); - realOutStream.Release(); + CMyComPtr2_Create outStream; + outStream->SetStream(realOutStream); + outStream->Init(); + // realOutStream.Release(); - CLocalProgress *lps = new CLocalProgress; - CMyComPtr progress = lps; + CMyComPtr2_Create lps; lps->Init(extractCallback, false); lps->InSize = _item.HeaderSize; - lps->OutSize = outStreamSpec->GetSize(); + lps->OutSize = outStream->GetSize(); RINOK(lps->SetCur()) CItem item = _item; @@ -367,7 +367,7 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, CMyComPtr inStream2; - UInt64 unpackSize = _item.GetSize() - (UInt32)8; + const UInt64 unpackSize = _item.GetSize() - (UInt32)8; if (_item.IsZlib()) { _decoderZlibSpec = new NCompress::NZlib::CDecoder; @@ -403,11 +403,11 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, RINOK(_decoderLzmaSpec->SetDecoderProperties2(props, 5)) } RINOK(item.WriteHeader(outStream)) - HRESULT result = _decoder->Code(inStream2, outStream, NULL, &unpackSize, progress); - Int32 opRes = NExtract::NOperationResult::kDataError; + const HRESULT result = _decoder->Code(inStream2, outStream, NULL, &unpackSize, lps); + opRes = NExtract::NOperationResult::kDataError; if (result == S_OK) { - if (item.GetSize() == outStreamSpec->GetSize()) + if (item.GetSize() == outStream->GetSize()) { if (_item.IsZlib()) { @@ -425,7 +425,8 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, else if (result != S_FALSE) return result; - outStream.Release(); + // outStream.Release(); + } return extractCallback->SetOperationResult(opRes); COM_TRY_END } @@ -488,11 +489,10 @@ static HRESULT UpdateArchive(ISequentialOutStream *outStream, UInt64 size, } RINOK(item.WriteHeader(outStream)) - CLocalProgress *lps = new CLocalProgress; - CMyComPtr progress = lps; + CMyComPtr2_Create lps; lps->Init(updateCallback, true); - RINOK(encoder->Code(fileInStream, outStream, NULL, NULL, progress)) + RINOK(encoder->Code(fileInStream, outStream, NULL, NULL, lps)) UInt64 inputProcessed; if (lzmaMode) { diff --git a/CPP/7zip/Archive/Tar/TarHandler.cpp b/CPP/7zip/Archive/Tar/TarHandler.cpp index d7fe175..29f28e8 100644 --- a/CPP/7zip/Archive/Tar/TarHandler.cpp +++ b/CPP/7zip/Archive/Tar/TarHandler.cpp @@ -39,6 +39,9 @@ static const Byte kProps[] = kpidCTime, kpidATime, kpidPosixAttrib, +#if 0 + kpidAttrib, +#endif kpidUser, kpidGroup, kpidUserId, @@ -79,13 +82,14 @@ Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) UInt32 flags = 0; if (!_isArc) flags |= kpv_ErrorFlags_IsNotArc; - else switch (_arc._error) + else switch ((int)_arc._error) { case k_ErrorType_UnexpectedEnd: flags = kpv_ErrorFlags_UnexpectedEnd; break; case k_ErrorType_Corrupted: flags = kpv_ErrorFlags_HeadersError; break; - case k_ErrorType_OK: break; + // case k_ErrorType_OK: break; // case k_ErrorType_Warning: break; - // default: break; + // case k_ErrorType_OK: + default: break; } if (flags != 0) prop = flags; @@ -107,6 +111,7 @@ Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) { case CP_OEMCP: name = "OEM"; break; case CP_UTF8: name = "UTF-8"; break; + default: break; } if (!name) { @@ -132,6 +137,7 @@ Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) if (_arc._are_mtime) s.Add_OptSpaced("mtime"); if (_arc._are_atime) s.Add_OptSpaced("atime"); if (_arc._are_ctime) s.Add_OptSpaced("ctime"); + if (_arc._are_SCHILY_fflags) s.Add_OptSpaced("SCHILY.fflags"); if (_arc._is_PaxGlobal_Error) s.Add_OptSpaced("PAX_GLOBAL_ERROR"); s.Add_OptSpaced(_encodingCharacts.GetCharactsString()); prop = s; @@ -149,6 +155,7 @@ Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) } break; } + default: break; } prop.Detach(value); return S_OK; @@ -354,8 +361,8 @@ Z7_COM7F_IMF(CHandler::GetNumberOfItems(UInt32 *numItems)) CHandler::CHandler() { - copyCoderSpec = new NCompress::CCopyCoder(); - copyCoder = copyCoderSpec; + // copyCoder = new NCompress::CCopyCoder(); + // copyCoder = copyCoder; _openCodePage = CP_UTF8; Init(); } @@ -367,9 +374,9 @@ HRESULT CHandler::SkipTo(UInt32 index) if (_latestIsRead) { const UInt64 packSize = _latestItem.Get_PackSize_Aligned(); - RINOK(copyCoder->Code(_seqStream, NULL, &packSize, &packSize, NULL)) - _arc._phySize += copyCoderSpec->TotalSize; - if (copyCoderSpec->TotalSize != packSize) + RINOK(copyCoder.Interface()->Code(_seqStream, NULL, &packSize, &packSize, NULL)) + _arc._phySize += copyCoder->TotalSize; + if (copyCoder->TotalSize != packSize) { _arc._error = k_ErrorType_UnexpectedEnd; return S_FALSE; @@ -423,27 +430,17 @@ static void PaxTimeToProp(const CPaxTime &pt, NWindows::NCOM::CPropVariant &prop } -#define ValToHex(t) ((char)(((t) < 10) ? ('0' + (t)) : ('a' + ((t) - 10)))) - -static void AddByteToHex2(unsigned val, AString &s) -{ - unsigned t; - t = val >> 4; - s += ValToHex(t); - t = val & 0xF; - s += ValToHex(t); -} - static void AddSpecCharToString(const char c, AString &s) { if ((Byte)c <= 0x20 || (Byte)c > 127) { - s += '['; - AddByteToHex2((Byte)(c), s); - s += ']'; + s.Add_Char('['); + s.Add_Char(GET_HEX_CHAR_LOWER((Byte)c >> 4)); + s.Add_Char(GET_HEX_CHAR_LOWER(c & 15)); + s.Add_Char(']'); } else - s += c; + s.Add_Char(c); } static void AddSpecUInt64(AString &s, const char *name, UInt64 v) @@ -453,7 +450,7 @@ static void AddSpecUInt64(AString &s, const char *name, UInt64 v) s.Add_OptSpaced(name); if (v > 1) { - s += ':'; + s.Add_Colon(); s.Add_UInt64(v); } } @@ -465,11 +462,36 @@ static void AddSpecBools(AString &s, const char *name, bool b1, bool b2) { s.Add_OptSpaced(name); if (b2) - s += '*'; + s.Add_Char('*'); } } +#if 0 +static bool Parse_Attrib_from_SCHILY_fflags(const AString &s, UInt32 &attribRes) +{ + UInt32 attrib = 0; + attribRes = attrib; + unsigned pos = 0; + while (pos < s.Len()) + { + int pos2 = s.Find(',', pos); + if (pos2 < 0) + pos2 = (int)s.Len(); + const AString str = s.Mid(pos, (unsigned)pos2 - pos); + if (str.IsEqualTo("hidden")) attrib |= FILE_ATTRIBUTE_HIDDEN; + else if (str.IsEqualTo("rdonly")) attrib |= FILE_ATTRIBUTE_READONLY; + else if (str.IsEqualTo("system")) attrib |= FILE_ATTRIBUTE_SYSTEM; + else + return false; + pos = (unsigned)pos2 + 1; + } + attribRes = attrib; + return true; +} +#endif + + Z7_COM7F_IMF(CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *value)) { COM_TRY_BEGIN @@ -535,7 +557,30 @@ Z7_COM7F_IMF(CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val PaxTimeToProp(item->PaxTimes.CTime, prop); break; case kpidPosixAttrib: prop = item->Get_Combined_Mode(); break; - + + // kpidAttrib has priority over kpidPosixAttrib in 7-Zip. + // but if we want kpidPosixAttrib priority for TAR, we disable kpidAttrib. +#if 0 + case kpidAttrib: + { + if (!item->SCHILY_fflags.IsEmpty()) + { + UInt32 attrib = 0; + if (Parse_Attrib_from_SCHILY_fflags(item->SCHILY_fflags, attrib)) + { + if (attrib != 0) + { + if (item->IsDir()) + attrib |= FILE_ATTRIBUTE_DIRECTORY; + attrib |= ((UInt32)item->Get_Combined_Mode() << 16) | 0x8000; // FILE_ATTRIBUTE_UNIX_EXTENSION; + prop = attrib; + } + } + } + break; + } +#endif + case kpidUser: if (!item->User.IsEmpty()) TarStringToUnicode(item->User, prop); @@ -634,7 +679,11 @@ Z7_COM7F_IMF(CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val s.Add_OptSpaced("pax_linkpath"); if (item->pax_size_WasUsed) s.Add_OptSpaced("pax_size"); - + if (!item->SCHILY_fflags.IsEmpty()) + { + s.Add_OptSpaced("SCHILY.fflags="); + s += item->SCHILY_fflags; + } if (item->IsThereWarning()) s.Add_OptSpaced("WARNING"); if (item->HeaderError) @@ -659,6 +708,7 @@ Z7_COM7F_IMF(CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val } // case kpidHeadersSize: prop = item->HeaderSize; break; // for debug // case kpidOffset: prop = item->HeaderPos; break; // for debug + default: break; } prop.Detach(value); return S_OK; @@ -684,36 +734,29 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, UInt32 i; for (i = 0; i < numItems; i++) totalSize += _items[allFilesMode ? i : indices[i]].Get_UnpackSize(); - extractCallback->SetTotal(totalSize); + RINOK(extractCallback->SetTotal(totalSize)) UInt64 totalPackSize; totalSize = totalPackSize = 0; - CLocalProgress *lps = new CLocalProgress; - CMyComPtr progress = lps; + CMyComPtr2_Create lps; lps->Init(extractCallback, false); + CMyComPtr2_Create inStream; + inStream->SetStream(stream); + CMyComPtr2_Create outStreamSpec; - CLimitedSequentialInStream *streamSpec = new CLimitedSequentialInStream; - CMyComPtr inStream(streamSpec); - streamSpec->SetStream(stream); - - CLimitedSequentialOutStream *outStreamSpec = new CLimitedSequentialOutStream; - CMyComPtr outStream(outStreamSpec); - - for (i = 0; i < numItems || seqMode; i++) + for (i = 0; ; i++) { lps->InSize = totalPackSize; lps->OutSize = totalSize; RINOK(lps->SetCur()) - CMyComPtr realOutStream; - Int32 askMode = testMode ? - NExtract::NAskMode::kTest : - NExtract::NAskMode::kExtract; + if (i >= numItems && !seqMode) + break; const UInt32 index = allFilesMode ? i : indices[i]; const CItemEx *item; if (seqMode) { - HRESULT res = SkipTo(index); + const HRESULT res = SkipTo(index); if (res == E_INVALIDARG) break; RINOK(res) @@ -722,6 +765,10 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, else item = &_items[index]; + Int32 askMode = testMode ? + NExtract::NAskMode::kTest : + NExtract::NAskMode::kExtract; + CMyComPtr realOutStream; RINOK(extractCallback->GetStream(index, &realOutStream, askMode)) const UInt64 unpackSize = item->Get_UnpackSize(); totalSize += unpackSize; @@ -729,6 +776,7 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, if (item->IsDir()) { RINOK(extractCallback->PrepareOperation(askMode)) + // realOutStream.Release(); RINOK(extractCallback->SetOperationResult(NExtract::NOperationResult::kOK)) continue; } @@ -738,9 +786,10 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, if (!seqMode) { /* - // probably we must show extracting info it callback handler instead - if (item->IsHardLink() || - item->IsSymLink()) + // GetStream() creates link. + // so we can show extracting info in GetStream() instead + if (item->Is_HardLink() || + item->Is_SymLink()) { RINOK(extractCallback->PrepareOperation(askMode)) RINOK(extractCallback->SetOperationResult(NExtract::NOperationResult::kOK)) @@ -779,8 +828,8 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, { RINOK(InStream_SeekSet(_stream, item->Get_DataPos())) } - streamSpec->Init(item->Get_PackSize_Aligned()); - RINOK(copyCoder->Code(inStream2, outStream, NULL, NULL, progress)) + inStream->Init(item->Get_PackSize_Aligned()); + RINOK(copyCoder.Interface()->Code(inStream2, outStreamSpec, NULL, NULL, lps)) } if (outStreamSpec->GetRem() != 0) opRes = NExtract::NOperationResult::kDataError; diff --git a/CPP/7zip/Archive/Tar/TarHandler.h b/CPP/7zip/Archive/Tar/TarHandler.h index 451c2fd..f1ef3b6 100644 --- a/CPP/7zip/Archive/Tar/TarHandler.h +++ b/CPP/7zip/Archive/Tar/TarHandler.h @@ -42,8 +42,7 @@ Z7_CLASS_IMP_CHandler_IInArchive_4( CArchive _arc; - NCompress::CCopyCoder *copyCoderSpec; - CMyComPtr copyCoder; + CMyComPtr2_Create copyCoder; HRESULT Open2(IInStream *stream, IArchiveOpenCallback *callback); HRESULT SkipTo(UInt32 index); diff --git a/CPP/7zip/Archive/Tar/TarHandlerOut.cpp b/CPP/7zip/Archive/Tar/TarHandlerOut.cpp index c93a86e..bcbb35d 100644 --- a/CPP/7zip/Archive/Tar/TarHandlerOut.cpp +++ b/CPP/7zip/Archive/Tar/TarHandlerOut.cpp @@ -257,8 +257,8 @@ Z7_COM7F_IMF(CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt RINOK(GetPropString(callback, i, kpidPath, ui.Name, codePage, utfFlags, true)) if (ui.IsDir && !ui.Name.IsEmpty() && ui.Name.Back() != '/') - ui.Name += '/'; - // ui.Name += '/'; // for debug + ui.Name.Add_Slash(); + // ui.Name.Add_Slash(); // for debug if (_posixMode) { diff --git a/CPP/7zip/Archive/Tar/TarIn.cpp b/CPP/7zip/Archive/Tar/TarIn.cpp index 90b6f84..22b8902 100644 --- a/CPP/7zip/Archive/Tar/TarIn.cpp +++ b/CPP/7zip/Archive/Tar/TarIn.cpp @@ -282,7 +282,7 @@ HRESULT CArchive::GetNextItemReal(CItemEx &item) { item.Prefix_WasUsed = true; ReadString(p, NFileHeader::kPrefixSize, item.Name); - item.Name += '/'; + item.Name.Add_Slash(); unsigned i; for (i = 0; i < NFileHeader::kNameSize; i++) if (buf[i] == 0) @@ -541,6 +541,7 @@ struct CPaxInfo: public CPaxTimes bool Link_Defined; bool User_Defined; bool Group_Defined; + bool SCHILY_fflags_Defined; UInt64 Size; UInt32 UID; @@ -551,6 +552,7 @@ struct CPaxInfo: public CPaxTimes AString User; AString Group; AString UnknownLines; + AString SCHILY_fflags; bool ParseID(const AString &val, bool &defined, UInt32 &res) { @@ -648,6 +650,7 @@ bool CPaxInfo::ParsePax(const CTempBuffer &tb, bool isFile) Link_Defined = false; User_Defined = false; Group_Defined = false; + SCHILY_fflags_Defined = false; // CPaxTimes::Clear(); @@ -759,6 +762,14 @@ bool CPaxInfo::ParsePax(const CTempBuffer &tb, bool isFile) { parsed = ParsePaxTime(val, ATime, DoubleTagError); } else if (name.IsEqualTo("ctime")) { parsed = ParsePaxTime(val, CTime, DoubleTagError); } + else if (name.IsEqualTo("SCHILY.fflags")) + { + if (SCHILY_fflags_Defined) + DoubleTagError = true; + SCHILY_fflags = val; + SCHILY_fflags_Defined = true; + parsed = true; + } else isDetectedName = false; if (isDetectedName && !parsed) @@ -812,6 +823,7 @@ HRESULT CArchive::ReadItem2(CItemEx &item) item.pax_size_WasUsed = false; item.PaxExtra.Clear(); + item.SCHILY_fflags.Empty(); item.EncodingCharacts.Clear(); @@ -1032,6 +1044,11 @@ HRESULT CArchive::ReadItem2(CItemEx &item) item.Group = paxInfo.Group; // item.pax_gname_WasUsed = true; } + if (paxInfo.SCHILY_fflags_Defined) + { + item.SCHILY_fflags = paxInfo.SCHILY_fflags; + // item.SCHILY_fflags_WasUsed = true; + } if (paxInfo.UID_Defined) { item.UID = (UInt32)paxInfo.UID; @@ -1098,6 +1115,7 @@ HRESULT CArchive::ReadItem(CItemEx &item) if (item.PaxTimes.MTime.IsDefined()) _are_mtime = true; if (item.PaxTimes.ATime.IsDefined()) _are_atime = true; if (item.PaxTimes.CTime.IsDefined()) _are_ctime = true; + if (!item.SCHILY_fflags.IsEmpty()) _are_SCHILY_fflags = true; if (item.pax_path_WasUsed) _are_pax_path = true; diff --git a/CPP/7zip/Archive/Tar/TarIn.h b/CPP/7zip/Archive/Tar/TarIn.h index 8c69057..edfc94a 100644 --- a/CPP/7zip/Archive/Tar/TarIn.h +++ b/CPP/7zip/Archive/Tar/TarIn.h @@ -61,6 +61,7 @@ class CArchive bool _are_LongName; bool _are_LongLink; bool _pathPrefix_WasUsed; + bool _are_SCHILY_fflags; // bool _isSparse; // temp internal vars for ReadItem(): @@ -110,6 +111,7 @@ class CArchive _are_LongName = false; _are_LongLink = false; _pathPrefix_WasUsed = false; + _are_SCHILY_fflags = false; // _isSparse = false; _is_Warning = false; diff --git a/CPP/7zip/Archive/Tar/TarItem.h b/CPP/7zip/Archive/Tar/TarItem.h index 2e12c9d..112f38d 100644 --- a/CPP/7zip/Archive/Tar/TarItem.h +++ b/CPP/7zip/Archive/Tar/TarItem.h @@ -161,6 +161,7 @@ struct CItem case NFileHeader::NLinkFlag::kPax_2: case NFileHeader::NLinkFlag::kGlobal: return true; + default: break; } return false; } @@ -195,6 +196,7 @@ struct CItem case NFileHeader::NLinkFlag::kCharacter: return MY_LIN_S_IFCHR; case NFileHeader::NLinkFlag::kFIFO: return MY_LIN_S_IFIFO; // case return MY_LIN_S_IFSOCK; + default: break; } if (IsDir()) @@ -218,6 +220,7 @@ struct CItem // we also do it return Name.Back() == '/'; // return NItemName::HasTailSlash(Name, CP_OEMCP); + default: break; } return false; } @@ -347,6 +350,7 @@ struct CItemEx: public CItem UInt64 Num_Pax_Records; CPaxExtra PaxExtra; + AString SCHILY_fflags; CEncodingCharacts EncodingCharacts; diff --git a/CPP/7zip/Archive/Tar/TarOut.cpp b/CPP/7zip/Archive/Tar/TarOut.cpp index 26d0855..8a80c0c 100644 --- a/CPP/7zip/Archive/Tar/TarOut.cpp +++ b/CPP/7zip/Archive/Tar/TarOut.cpp @@ -232,7 +232,7 @@ static void AddPaxLine(AString &s, const char *name, const AString &val) s += n; s.Add_Space(); s += name; - s += '='; + s.Add_Char('='); s += val; s.Add_LF(); } diff --git a/CPP/7zip/Archive/Tar/TarUpdate.cpp b/CPP/7zip/Archive/Tar/TarUpdate.cpp index 0b348e9..d4ed772 100644 --- a/CPP/7zip/Archive/Tar/TarUpdate.cpp +++ b/CPP/7zip/Archive/Tar/TarUpdate.cpp @@ -184,23 +184,23 @@ HRESULT UpdateArchive(IInStream *inStream, ISequentialOutStream *outStream, { const CUpdateItem &ui = updateItems[i]; if (ui.NewData) + { + if (ui.Size == (UInt64)(Int64)-1) + break; complexity += ui.Size; + } else complexity += inputItems[(unsigned)ui.IndexInArc].Get_FullSize_Aligned(); } - RINOK(updateCallback->SetTotal(complexity)) + if (i == updateItems.Size()) + RINOK(updateCallback->SetTotal(complexity)) - NCompress::CCopyCoder *copyCoderSpec = new NCompress::CCopyCoder; - CMyComPtr copyCoder = copyCoderSpec; - - CLocalProgress *lps = new CLocalProgress; - CMyComPtr progress = lps; + CMyComPtr2_Create lps; lps->Init(updateCallback, true); - - CLimitedSequentialInStream *streamSpec = new CLimitedSequentialInStream; - CMyComPtr inStreamLimited(streamSpec); - streamSpec->SetStream(inStream); + CMyComPtr2_Create copyCoder; + CMyComPtr2_Create inStreamLimited; + inStreamLimited->SetStream(inStream); complexity = 0; @@ -271,9 +271,10 @@ HRESULT UpdateArchive(IInStream *inStream, ISequentialOutStream *outStream, item.SparseBlocks.Clear(); item.PackSize = ui.Size; item.Size = ui.Size; +#if 0 if (ui.Size == (UInt64)(Int64)-1) return E_INVALIDARG; - +#endif CMyComPtr fileInStream; bool needWrite = true; @@ -353,6 +354,7 @@ HRESULT UpdateArchive(IInStream *inStream, ISequentialOutStream *outStream, if (prop.vt != VT_UI8) return E_INVALIDARG; const UInt64 size = prop.uhVal.QuadPart; + // printf("\nTAR after GetProperty(kpidSize size = %8d\n", (unsigned)size); item.PackSize = size; item.Size = size; } @@ -409,6 +411,11 @@ HRESULT UpdateArchive(IInStream *inStream, ISequentialOutStream *outStream, if (needWrite) { + if (fileInStream) + // if (item.PackSize == (UInt64)(Int64)-1) + if (item.Size == (UInt64)(Int64)-1) + return E_INVALIDARG; + const UInt64 headerPos = outArchive.Pos; // item.PackSize = ((UInt64)1 << 33); // for debug @@ -420,6 +427,7 @@ HRESULT UpdateArchive(IInStream *inStream, ISequentialOutStream *outStream, { for (unsigned numPasses = 0;; numPasses++) { + // printf("\nTAR numPasses = %d" " old size = %8d\n", numPasses, (unsigned)item.PackSize); /* we support 2 attempts to write header: pass-0: main pass: pass-1: additional pass, if size_of_file and size_of_header are changed */ @@ -431,12 +439,12 @@ HRESULT UpdateArchive(IInStream *inStream, ISequentialOutStream *outStream, } const UInt64 dataPos = outArchive.Pos; - RINOK(copyCoder->Code(fileInStream, outStream, NULL, NULL, progress)) - outArchive.Pos += copyCoderSpec->TotalSize; - RINOK(outArchive.Write_AfterDataResidual(copyCoderSpec->TotalSize)) - + RINOK(copyCoder.Interface()->Code(fileInStream, outStream, NULL, NULL, lps)) + outArchive.Pos += copyCoder->TotalSize; + RINOK(outArchive.Write_AfterDataResidual(copyCoder->TotalSize)) + // printf("\nTAR after Code old size = %8d copyCoder->TotalSize = %8d \n", (unsigned)item.PackSize, (unsigned)copyCoder->TotalSize); // if (numPasses >= 10) // for debug - if (copyCoderSpec->TotalSize == item.PackSize) + if (copyCoder->TotalSize == item.PackSize) break; if (opCallback) @@ -451,7 +459,7 @@ HRESULT UpdateArchive(IInStream *inStream, ISequentialOutStream *outStream, const UInt64 nextPos = outArchive.Pos; RINOK(outSeekStream->Seek(-(Int64)(nextPos - headerPos), STREAM_SEEK_CUR, NULL)) outArchive.Pos = headerPos; - item.PackSize = copyCoderSpec->TotalSize; + item.PackSize = copyCoder->TotalSize; RINOK(outArchive.WriteHeader(item)) @@ -537,12 +545,12 @@ HRESULT UpdateArchive(IInStream *inStream, ISequentialOutStream *outStream, if (size != 0) { RINOK(InStream_SeekSet(inStream, pos)) - streamSpec->Init(size); + inStreamLimited->Init(size); if (outSeekStream && setRestriction) RINOK(setRestriction->SetRestriction(0, 0)) // 22.00 : we copy Residual data from old archive to new archive instead of zeroing - RINOK(copyCoder->Code(inStreamLimited, outStream, NULL, NULL, progress)) - if (copyCoderSpec->TotalSize != size) + RINOK(copyCoder.Interface()->Code(inStreamLimited, outStream, NULL, NULL, lps)) + if (copyCoder->TotalSize != size) return E_FAIL; outArchive.Pos += size; // RINOK(outArchive.Write_AfterDataResidual(existItem.PackSize)); diff --git a/CPP/7zip/Archive/Udf/UdfHandler.cpp b/CPP/7zip/Archive/Udf/UdfHandler.cpp index ae6113f..7c74c80 100644 --- a/CPP/7zip/Archive/Udf/UdfHandler.cpp +++ b/CPP/7zip/Archive/Udf/UdfHandler.cpp @@ -332,7 +332,7 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, for (i = 0; i < numItems; i++) { - UInt32 index = (allFilesMode ? i : indices[i]); + const UInt32 index = (allFilesMode ? i : indices[i]); const CRef2 &ref2 = _refs2[index]; const CRef &ref = _archive.LogVols[ref2.Vol].FileSets[ref2.Fs].Refs[ref2.Ref]; const CFile &file = _archive.Files[ref.FileIndex]; @@ -340,24 +340,21 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, if (!item.IsDir()) totalSize += item.Size; } - extractCallback->SetTotal(totalSize); + RINOK(extractCallback->SetTotal(totalSize)) UInt64 currentTotalSize = 0; - NCompress::CCopyCoder *copyCoderSpec = new NCompress::CCopyCoder(); - CMyComPtr copyCoder = copyCoderSpec; - - CLocalProgress *lps = new CLocalProgress; - CMyComPtr progress = lps; + CMyComPtr2_Create lps; lps->Init(extractCallback, false); + CMyComPtr2_Create copyCoder; + CMyComPtr2_Create outStream; - CLimitedSequentialOutStream *outStreamSpec = new CLimitedSequentialOutStream; - CMyComPtr outStream(outStreamSpec); - - for (i = 0; i < numItems; i++) + for (i = 0;; i++) { lps->InSize = lps->OutSize = currentTotalSize; RINOK(lps->SetCur()) + if (i >= numItems) + break; CMyComPtr realOutStream; const Int32 askMode = testMode ? NExtract::NAskMode::kTest : @@ -383,24 +380,26 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, continue; RINOK(extractCallback->PrepareOperation(askMode)) - outStreamSpec->SetStream(realOutStream); + outStream->SetStream(realOutStream); realOutStream.Release(); - outStreamSpec->Init(item.Size); + outStream->Init(item.Size); Int32 opRes; - CMyComPtr udfInStream; - HRESULT res = GetStream(index, &udfInStream); - if (res == E_NOTIMPL) - opRes = NExtract::NOperationResult::kUnsupportedMethod; - else if (res != S_OK) - opRes = NExtract::NOperationResult::kDataError; - else { - RINOK(copyCoder->Code(udfInStream, outStream, NULL, NULL, progress)) - opRes = outStreamSpec->IsFinishedOK() ? - NExtract::NOperationResult::kOK: - NExtract::NOperationResult::kDataError; + CMyComPtr udfInStream; + const HRESULT res = GetStream(index, &udfInStream); + if (res == E_NOTIMPL) + opRes = NExtract::NOperationResult::kUnsupportedMethod; + else if (res != S_OK) + opRes = NExtract::NOperationResult::kDataError; + else + { + RINOK(copyCoder.Interface()->Code(udfInStream, outStream, NULL, NULL, lps)) + opRes = outStream->IsFinishedOK() ? + NExtract::NOperationResult::kOK: + NExtract::NOperationResult::kDataError; + } } - outStreamSpec->ReleaseStream(); + outStream->ReleaseStream(); RINOK(extractCallback->SetOperationResult(opRes)) } return S_OK; diff --git a/CPP/7zip/Archive/Udf/UdfIn.cpp b/CPP/7zip/Archive/Udf/UdfIn.cpp index 13d84e0..ce87c54 100644 --- a/CPP/7zip/Archive/Udf/UdfIn.cpp +++ b/CPP/7zip/Archive/Udf/UdfIn.cpp @@ -1731,7 +1731,7 @@ UString CInArchive::GetItemPath(unsigned volIndex, unsigned fsIndex, unsigned re UString newName2 = vol.GetName(); if (newName2.IsEmpty()) newName2 = "Volume"; - newName += '-'; + newName.Add_Minus(); newName += newName2; UpdateWithName(name, newName); } diff --git a/CPP/7zip/Archive/UefiHandler.cpp b/CPP/7zip/Archive/UefiHandler.cpp index cd6fcbd..9462e97 100644 --- a/CPP/7zip/Archive/UefiHandler.cpp +++ b/CPP/7zip/Archive/UefiHandler.cpp @@ -10,9 +10,10 @@ #include "../../../C/7zCrc.h" #include "../../../C/Alloc.h" -#include "../../../C/CpuArch.h" #include "../../../C/LzmaDec.h" +#include "../../../C/CpuArch.h" +#include "../../Common/AutoPtr.h" #include "../../Common/ComTry.h" #include "../../Common/IntToString.h" #include "../../Common/MyBuffer.h" @@ -42,8 +43,8 @@ namespace NArchive { namespace NUefi { -static const size_t kBufTotalSizeMax = (1 << 29); -static const unsigned kNumFilesMax = (1 << 18); +static const size_t kBufTotalSizeMax = 1 << 29; +static const unsigned kNumFilesMax = 1 << 18; static const unsigned kLevelMax = 64; static const Byte k_IntelMeSignature[] = @@ -845,7 +846,7 @@ Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) case kpidErrorFlags: { UInt32 v = 0; - if (!_headersError) v |= kpv_ErrorFlags_HeadersError; + if (_headersError) v |= kpv_ErrorFlags_HeadersError; if (v != 0) prop = v; break; @@ -990,8 +991,8 @@ HRESULT CHandler::ParseSections(unsigned bufIndex, UInt32 posBase, UInt32 size, { if (sectSize < 4 + 5) return S_FALSE; - UInt32 uncompressedSize = Get32(p + 4); - Byte compressionType = p[8]; + const UInt32 uncompressedSize = Get32(p + 4); + const Byte compressionType = p[8]; UInt32 newSectSize = sectSize - 9; UInt32 newOffset = posBase + pos + 9; @@ -1014,21 +1015,16 @@ HRESULT CHandler::ParseSections(unsigned bufIndex, UInt32 posBase, UInt32 size, } else if (compressionType == COMPRESSION_TYPE_LZH) { - unsigned newBufIndex = AddBuf(uncompressedSize); + const unsigned newBufIndex = AddBuf(uncompressedSize); CByteBuffer &buf = _bufs[newBufIndex]; - - NCompress::NLzh::NDecoder::CCoder *lzhDecoderSpec = NULL; - CMyComPtr lzhDecoder; - - lzhDecoderSpec = new NCompress::NLzh::NDecoder::CCoder; - lzhDecoder = lzhDecoderSpec; - + CMyUniquePtr lzhDecoder; + lzhDecoder.Create_if_Empty(); { const Byte *src = pStart; if (newSectSize < 8) return S_FALSE; UInt32 packSize = Get32(src); - UInt32 unpackSize = Get32(src + 4); + const UInt32 unpackSize = Get32(src + 4); PRF(printf(" LZH packSize = %6x, unpackSize = %6x", packSize, unpackSize)); @@ -1041,14 +1037,10 @@ HRESULT CHandler::ParseSections(unsigned bufIndex, UInt32 posBase, UInt32 size, if (src[packSize] != 0) return S_FALSE; - CBufInStream *inStreamSpec = new CBufInStream; - CMyComPtr inStream = inStreamSpec; - - CBufPtrSeqOutStream *outStreamSpec = new CBufPtrSeqOutStream; - CMyComPtr outStream = outStreamSpec; - - UInt64 uncompressedSize64 = uncompressedSize; - lzhDecoderSpec->FinishMode = true; + CMyComPtr2_Create inStream; + CMyComPtr2_Create outStream; + // const UInt64 uncompressedSize64 = uncompressedSize; + // lzhDecoder->FinishMode = true; /* EFI 1.1 probably used LZH with small dictionary and (pbit = 4). It was named "Efi compression". New version of compression code (named Tiano) uses LZH with (1 << 19) dictionary. @@ -1056,19 +1048,17 @@ HRESULT CHandler::ParseSections(unsigned bufIndex, UInt32 posBase, UInt32 size, We check both LZH versions: Tiano and then Efi. */ HRESULT res = S_FALSE; - for (unsigned m = 0 ; m < 2; m++) { - inStreamSpec->Init(src, packSize); - outStreamSpec->Init(buf, uncompressedSize); - lzhDecoderSpec->SetDictSize((m == 0) ? ((UInt32)1 << 19) : ((UInt32)1 << 14)); - res = lzhDecoder->Code(inStream, outStream, NULL, &uncompressedSize64, NULL); + inStream->Init(src, packSize); + outStream->Init(buf, uncompressedSize); + lzhDecoder->SetDictSize(m == 0 ? ((UInt32)1 << 19) : ((UInt32)1 << 14)); + res = lzhDecoder->Code(inStream, outStream, uncompressedSize, NULL); if (res == S_OK) break; } RINOK(res) } - bool error2; RINOK(ParseSections(newBufIndex, 0, uncompressedSize, parent, compressionType, level, error2)) } @@ -1781,49 +1771,48 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, UInt32 i; for (i = 0; i < numItems; i++) totalSize += _items[_items2[allFilesMode ? i : indices[i]].MainIndex].Size; - extractCallback->SetTotal(totalSize); - - UInt64 currentTotalSize = 0; + RINOK(extractCallback->SetTotal(totalSize)) + totalSize = 0; - NCompress::CCopyCoder *copyCoderSpec = new NCompress::CCopyCoder(); - CMyComPtr copyCoder = copyCoderSpec; - - CLocalProgress *lps = new CLocalProgress; - CMyComPtr progress = lps; + CMyComPtr2_Create lps; lps->Init(extractCallback, false); + CMyComPtr2_Create copyCoder; - for (i = 0; i < numItems; i++) + for (i = 0;; i++) { - lps->InSize = lps->OutSize = currentTotalSize; + lps->InSize = lps->OutSize = totalSize; RINOK(lps->SetCur()) - CMyComPtr realOutStream; - const Int32 askMode = testMode ? - NExtract::NAskMode::kTest : - NExtract::NAskMode::kExtract; - const UInt32 index = allFilesMode ? i : indices[i]; - const CItem &item = _items[_items2[index].MainIndex]; - RINOK(extractCallback->GetStream(index, &realOutStream, askMode)) - currentTotalSize += item.Size; - - if (!testMode && !realOutStream) - continue; - RINOK(extractCallback->PrepareOperation(askMode)) - if (testMode || item.IsDir) - { - RINOK(extractCallback->SetOperationResult(NExtract::NOperationResult::kOK)) - continue; - } - int res = NExtract::NOperationResult::kDataError; - CMyComPtr inStream; - GetStream(index, &inStream); - if (inStream) + if (i >= numItems) + break; + Int32 opRes; { - RINOK(copyCoder->Code(inStream, realOutStream, NULL, NULL, progress)) - if (copyCoderSpec->TotalSize == item.Size) - res = NExtract::NOperationResult::kOK; + CMyComPtr realOutStream; + const Int32 askMode = testMode ? + NExtract::NAskMode::kTest : + NExtract::NAskMode::kExtract; + const UInt32 index = allFilesMode ? i : indices[i]; + const CItem &item = _items[_items2[index].MainIndex]; + RINOK(extractCallback->GetStream(index, &realOutStream, askMode)) + totalSize += item.Size; + if (!testMode && !realOutStream) + continue; + RINOK(extractCallback->PrepareOperation(askMode)) + if (testMode || item.IsDir) + opRes = NExtract::NOperationResult::kOK; + else + { + opRes = NExtract::NOperationResult::kDataError; + CMyComPtr inStream; + GetStream(index, &inStream); + if (inStream) + { + RINOK(copyCoder.Interface()->Code(inStream, realOutStream, NULL, NULL, lps)) + if (copyCoder->TotalSize == item.Size) + opRes = NExtract::NOperationResult::kOK; + } + } } - realOutStream.Release(); - RINOK(extractCallback->SetOperationResult(res)) + RINOK(extractCallback->SetOperationResult(opRes)) } return S_OK; COM_TRY_END diff --git a/CPP/7zip/Archive/VhdHandler.cpp b/CPP/7zip/Archive/VhdHandler.cpp index e5bf997..2fef079 100644 --- a/CPP/7zip/Archive/VhdHandler.cpp +++ b/CPP/7zip/Archive/VhdHandler.cpp @@ -5,6 +5,7 @@ #include "../../../C/CpuArch.h" #include "../../Common/ComTry.h" +#include "../../Common/IntToString.h" #include "../../Windows/PropVariant.h" @@ -278,9 +279,9 @@ Z7_class_CHandler_final: public CHandlerImg if (mainName != anotherName && !anotherName.IsEmpty()) { res.Add_Space(); - res += '('; + res.Add_Char('('); res += anotherName; - res += ')'; + res.Add_Char(')'); } p = p->Parent; } @@ -689,16 +690,6 @@ static void StringToAString(char *dest, UInt32 val) *dest = 0; } -static void ConvertByteToHex(unsigned value, char *s) -{ - for (int i = 0; i < 2; i++) - { - unsigned t = value & 0xF; - value >>= 4; - s[1 - i] = (char)((t < 10) ? ('0' + t) : ('A' + (t - 10))); - } -} - Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) { COM_TRY_BEGIN @@ -754,10 +745,8 @@ Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) } case kpidId: { - char s[32 + 4]; - for (int i = 0; i < 16; i++) - ConvertByteToHex(Footer.Id[i], s + i * 2); - s[32] = 0; + char s[sizeof(Footer.Id) * 2 + 2]; + ConvertDataToHex_Upper(s, Footer.Id, sizeof(Footer.Id)); prop = s; break; } @@ -949,13 +938,13 @@ Z7_COM7F_IMF(CHandler::GetStream(UInt32 /* index */, ISequentialInStream **strea *stream = NULL; if (Footer.IsFixed()) { - CLimitedInStream *streamSpec = new CLimitedInStream; - CMyComPtr streamTemp = streamSpec; + CMyComPtr2 streamSpec; + streamSpec.Create_if_Empty(); streamSpec->SetStream(Stream); // fixme : check (startOffset = 0) streamSpec->InitAndSeek(_startOffset, Footer.CurrentSize); RINOK(streamSpec->SeekToStart()) - *stream = streamTemp.Detach(); + *stream = streamSpec.Detach(); return S_OK; } if (!Footer.ThereIsDynamic() || !AreParentsOK()) diff --git a/CPP/7zip/Archive/VhdxHandler.cpp b/CPP/7zip/Archive/VhdxHandler.cpp index e3ddedd..ca450e5 100644 --- a/CPP/7zip/Archive/VhdxHandler.cpp +++ b/CPP/7zip/Archive/VhdxHandler.cpp @@ -7,6 +7,8 @@ #include "../../../C/CpuArch.h" #include "../../Common/ComTry.h" +#include "../../Common/IntToString.h" +#include "../../Common/StringToInt.h" #include "../../Common/MyBuffer.h" #include "../../Windows/PropVariant.h" @@ -31,6 +33,7 @@ EXTERN_C_BEGIN // CRC-32C (Castagnoli) : reversed for poly 0x1EDC6F41 #define k_Crc32c_Poly 0x82f63b78 +MY_ALIGN(64) static UInt32 g_Crc32c_Table[256]; static void Z7_FASTCALL Crc32c_GenerateTable() @@ -91,31 +94,13 @@ static bool IsZeroArr(const Byte *p, size_t size) } -#define ValToHex(t) ((char)(((t) < 10) ? ('0' + (t)) : ('a' + ((t) - 10)))) - -static void AddByteToHex2(unsigned val, UString &s) -{ - unsigned t; - t = val >> 4; - s += ValToHex(t); - t = val & 0xF; - s += ValToHex(t); -} - - -static int HexToVal(const wchar_t c) -{ - if (c >= '0' && c <= '9') return c - '0'; - if (c >= 'a' && c <= 'z') return c - 'a' + 10; - if (c >= 'A' && c <= 'Z') return c - 'A' + 10; - return -1; -} +Z7_FORCE_INLINE static int DecodeFrom2HexChars(const wchar_t *s) { - const int v0 = HexToVal(s[0]); if (v0 < 0) return -1; - const int v1 = HexToVal(s[1]); if (v1 < 0) return -1; - return (int)(((unsigned)v0 << 4) | (unsigned)v1); + unsigned v0 = (unsigned)s[0]; Z7_PARSE_HEX_DIGIT(v0, return -1;) + unsigned v1 = (unsigned)s[1]; Z7_PARSE_HEX_DIGIT(v1, return -1;) + return (int)((v0 << 4) | v1); } @@ -161,8 +146,9 @@ struct CGuid void CGuid::AddHexToString(UString &s) const { - for (unsigned i = 0; i < 16; i++) - AddByteToHex2(Data[i], s); + char temp[sizeof(Data) * 2 + 2]; + ConvertDataToHex_Lower(temp, Data, sizeof(Data)); + s += temp; } @@ -274,7 +260,7 @@ struct CRegion }; -static const unsigned kRegionSize = 1 << 16; +static const size_t kRegionSize = 1 << 16; static const unsigned kNumRegionEntriesMax = (1 << 11) - 1; bool CRegion::Parse(Byte *p) @@ -1576,7 +1562,7 @@ static void AddComment_Name(UString &s, const char *name) static void AddComment_Bool(UString &s, const char *name, bool val) { AddComment_Name(s, name); - s += val ? "+" : "-"; + s.Add_Char(val ? '+' : '-'); s.Add_LF(); } diff --git a/CPP/7zip/Archive/VmdkHandler.cpp b/CPP/7zip/Archive/VmdkHandler.cpp index 0a6b967..9c293a3 100644 --- a/CPP/7zip/Archive/VmdkHandler.cpp +++ b/CPP/7zip/Archive/VmdkHandler.cpp @@ -815,7 +815,7 @@ Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) bool zlib = false; bool marker = false; - int algo = -1; + Int32 algo = -1; FOR_VECTOR (i, _extents) { @@ -829,7 +829,7 @@ Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) { if (h.algo == 1) zlib = true; - else if (algo != (int)h.algo) + else if (algo != h.algo) { s.Add_Space_if_NotEmpty(); s.Add_UInt32(h.algo); diff --git a/CPP/7zip/Archive/Wim/WimHandler.cpp b/CPP/7zip/Archive/Wim/WimHandler.cpp index 7f96dcc..b21f673 100644 --- a/CPP/7zip/Archive/Wim/WimHandler.cpp +++ b/CPP/7zip/Archive/Wim/WimHandler.cpp @@ -299,7 +299,7 @@ Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) if (numMethods == 1 && chunkSizeBits != 0) { - res += ':'; + res.Add_Colon(); res.Add_UInt32((UInt32)chunkSizeBits); } @@ -1055,27 +1055,23 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, RINOK(extractCallback->SetTotal(totalSize)) - UInt64 currentTotalUnPacked = 0; + totalSize = 0; UInt64 currentItemUnPacked; int prevSuccessStreamIndex = -1; CUnpacker unpacker; - CLocalProgress *lps = new CLocalProgress; - CMyComPtr progress = lps; + CMyComPtr2_Create lps; lps->Init(extractCallback, false); for (i = 0;; i++, - currentTotalUnPacked += currentItemUnPacked) + totalSize += currentItemUnPacked) { currentItemUnPacked = 0; - lps->InSize = unpacker.TotalPacked; - lps->OutSize = currentTotalUnPacked; - + lps->OutSize = totalSize; RINOK(lps->SetCur()) - if (i >= numItems) break; @@ -1108,7 +1104,7 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, } const CItem &item = _db.Items[_db.SortedItems[index]]; - int streamIndex = item.StreamIndex; + const int streamIndex = item.StreamIndex; if (streamIndex < 0) { if (!item.IsDir) @@ -1135,10 +1131,9 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, { Byte digest[kHashSize]; const CVolume &vol = _volumes[si.PartNumber]; - bool needDigest = !si.IsEmptyHash(); - - HRESULT res = unpacker.Unpack(vol.Stream, si.Resource, vol.Header, &_db, - realOutStream, progress, needDigest ? digest : NULL); + const bool needDigest = !si.IsEmptyHash() && !_disable_Sha1Check; + const HRESULT res = unpacker.Unpack(vol.Stream, si.Resource, vol.Header, &_db, + realOutStream, lps, needDigest ? digest : NULL); if (res == S_OK) { @@ -1216,6 +1211,13 @@ Z7_COM7F_IMF(CHandler::SetProperties(const wchar_t * const *names, const PROPVAR else if (name.IsPrefixedBy_Ascii_NoCase("memuse")) { } + else if (name.IsPrefixedBy_Ascii_NoCase("crc")) + { + name.Delete(0, 3); + UInt32 crcSize = 1; + RINOK(ParsePropToUInt32(name, prop, crcSize)) + _disable_Sha1Check = (crcSize == 0); + } else { bool processed = false; diff --git a/CPP/7zip/Archive/Wim/WimHandler.h b/CPP/7zip/Archive/Wim/WimHandler.h index 60bb156..126d233 100644 --- a/CPP/7zip/Archive/Wim/WimHandler.h +++ b/CPP/7zip/Archive/Wim/WimHandler.h @@ -12,7 +12,7 @@ namespace NArchive { namespace NWim { -static const Int32 kNumImagesMaxUpdate = (1 << 10); +const Int32 kNumImagesMaxUpdate = 1 << 10; Z7_CLASS_IMP_CHandler_IInArchive_5( IArchiveGetRawProps @@ -23,17 +23,18 @@ Z7_CLASS_IMP_CHandler_IInArchive_5( ) CDatabase _db; UInt32 _version; - bool _isOldVersion; UInt32 _bootIndex; CObjectVector _volumes; CObjectVector _xmls; // unsigned _nameLenForStreams; - bool _xmlInComments; - + unsigned _numXmlItems; unsigned _numIgnoreItems; + bool _isOldVersion; + bool _xmlInComments; + bool _xmlError; bool _isArc; bool _unsupported; @@ -43,16 +44,17 @@ Z7_CLASS_IMP_CHandler_IInArchive_5( int _defaultImageNumber; bool _showImageNumber; - bool _keepMode_ShowImageNumber; + bool _disable_Sha1Check; UInt64 _phySize; - int _firstVolumeIndex; + Int32 _firstVolumeIndex; CHandlerTimeOptions _timeOptions; void InitDefaults() { + _disable_Sha1Check = false; _set_use_ShowImageNumber = false; _set_showImageNumber = false; _defaultImageNumber = -1; diff --git a/CPP/7zip/Archive/Wim/WimHandlerOut.cpp b/CPP/7zip/Archive/Wim/WimHandlerOut.cpp index 5d9dea2..3b2cb31 100644 --- a/CPP/7zip/Archive/Wim/WimHandlerOut.cpp +++ b/CPP/7zip/Archive/Wim/WimHandlerOut.cpp @@ -20,6 +20,8 @@ #include "../../Crypto/RandGen.h" #include "../../Crypto/Sha1Cls.h" +#include "../Common/OutStreamWithSha1.h" + #include "WimHandler.h" using namespace NWindows; @@ -345,40 +347,6 @@ void CStreamInfo::WriteTo(Byte *p) const } -Z7_CLASS_IMP_NOQIB_1( - CInStreamWithSha1 - , ISequentialInStream -) - CMyComPtr _stream; - UInt64 _size; - // NCrypto::NSha1::CContext _sha; - CAlignedBuffer1 _sha; - CSha1 *Sha() { return (CSha1 *)(void *)(Byte *)_sha; } -public: - CInStreamWithSha1(): _sha(sizeof(CSha1)) {} - void SetStream(ISequentialInStream *stream) { _stream = stream; } - void Init() - { - _size = 0; - Sha1_Init(Sha()); - } - void ReleaseStream() { _stream.Release(); } - UInt64 GetSize() const { return _size; } - void Final(Byte *digest) { Sha1_Final(Sha(), digest); } -}; - -Z7_COM7F_IMF(CInStreamWithSha1::Read(void *data, UInt32 size, UInt32 *processedSize)) -{ - UInt32 realProcessedSize; - const HRESULT result = _stream->Read(data, size, &realProcessedSize); - _size += realProcessedSize; - Sha1_Update(Sha(), (const Byte *)data, realProcessedSize); - if (processedSize) - *processedSize = realProcessedSize; - return result; -} - - static void SetFileTimeToMem(Byte *p, const FILETIME &ft) { Set32(p, ft.dwLowDateTime) @@ -599,14 +567,14 @@ void CDb::WriteOrderList(const CDir &tree) static void AddTag_ToString(AString &s, const char *name, const char *value) { - s += '<'; + s.Add_Char('<'); s += name; - s += '>'; + s.Add_Char('>'); s += value; - s += '<'; - s += '/'; + s.Add_Char('<'); + s.Add_Slash(); s += name; - s += '>'; + s.Add_Char('>'); } @@ -620,7 +588,7 @@ static void AddTagUInt64_ToString(AString &s, const char *name, UInt64 value) static CXmlItem &AddUniqueTag(CXmlItem &parentItem, const char *name) { - int index = parentItem.FindSubTag(name); + const int index = parentItem.FindSubTag(name); if (index < 0) { CXmlItem &subItem = parentItem.SubItems.AddNew(); @@ -679,8 +647,7 @@ static void AddTag_Time(CXmlItem &parentItem, const char *name, const FILETIME & static void AddTag_String_IfEmpty(CXmlItem &parentItem, const char *name, const char *value) { - int index = parentItem.FindSubTag(name); - if (index >= 0) + if (parentItem.FindSubTag(name) >= 0) return; CXmlItem &tag = parentItem.SubItems.AddNew(); tag.IsTag = true; @@ -1094,7 +1061,7 @@ Z7_COM7F_IMF(CHandler::UpdateItems(ISequentialOutStream *outSeqStream, UInt32 nu { for (;;) { - wchar_t c = *path++; + const wchar_t c = *path++; if (c == 0) break; if (c == WCHAR_PATH_SEPARATOR || c == L'/') @@ -1382,12 +1349,9 @@ Z7_COM7F_IMF(CHandler::UpdateItems(ISequentialOutStream *outSeqStream, UInt32 nu RINOK(callback->SetTotal(complexity)) UInt64 totalComplexity = complexity; - NCompress::CCopyCoder *copyCoderSpec = new NCompress::CCopyCoder; - CMyComPtr copyCoder = copyCoderSpec; - - CLocalProgress *lps = new CLocalProgress; - CMyComPtr progress = lps; + CMyComPtr2_Create lps; lps->Init(callback, true); + CMyComPtr2_Create copyCoder; complexity = 0; @@ -1419,8 +1383,7 @@ Z7_COM7F_IMF(CHandler::UpdateItems(ISequentialOutStream *outSeqStream, UInt32 nu UInt64 curPos = kHeaderSizeMax; - CInStreamWithSha1 *inShaStreamSpec = new CInStreamWithSha1; - CMyComPtr inShaStream = inShaStreamSpec; + CMyComPtr2_Create inShaStream; CLimitedSequentialInStream *inStreamLimitedSpec = NULL; CMyComPtr inStreamLimited; @@ -1492,7 +1455,7 @@ Z7_COM7F_IMF(CHandler::UpdateItems(ISequentialOutStream *outSeqStream, UInt32 nu if (!rs.IsSolid() || rs.IsSolidSmall()) { - const int find = AddUniqHash(&streams.Front(), sortedHashes, siOld.Hash, (int)streamIndex); + const int find = AddUniqHash(streams.ConstData(), sortedHashes, siOld.Hash, (int)streamIndex); if (find != -1) return E_FAIL; // two streams with same SHA-1 } @@ -1501,8 +1464,8 @@ Z7_COM7F_IMF(CHandler::UpdateItems(ISequentialOutStream *outSeqStream, UInt32 nu { RINOK(InStream_SeekSet(_volumes[siOld.PartNumber].Stream, rs.Offset)) inStreamLimitedSpec->Init(rs.PackSize); - RINOK(copyCoder->Code(inStreamLimited, outStream, NULL, NULL, progress)) - if (copyCoderSpec->TotalSize != rs.PackSize) + RINOK(copyCoder.Interface()->Code(inStreamLimited, outStream, NULL, NULL, lps)) + if (copyCoder->TotalSize != rs.PackSize) return E_FAIL; s.Resource.Offset = curPos; curPos += rs.PackSize; @@ -1564,7 +1527,7 @@ Z7_COM7F_IMF(CHandler::UpdateItems(ISequentialOutStream *outSeqStream, UInt32 nu const CStreamInfo &siOld = _db.DataStreams[item.StreamIndex]; - const int index = AddUniqHash(&streams.Front(), sortedHashes, siOld.Hash, -1); + const int index = AddUniqHash(streams.ConstData(), sortedHashes, siOld.Hash, -1); // we must have written that stream already if (index == -1) return E_FAIL; @@ -1655,7 +1618,7 @@ Z7_COM7F_IMF(CHandler::UpdateItems(ISequentialOutStream *outSeqStream, UInt32 nu Byte hash[kHashSize]; sha1.Final(hash); - int index = AddUniqHash(&streams.Front(), sortedHashes, hash, (int)streams.Size()); + int index = AddUniqHash(streams.ConstData(), sortedHashes, hash, (int)streams.Size()); if (index != -1) streams[index].RefCount++; @@ -1684,13 +1647,13 @@ Z7_COM7F_IMF(CHandler::UpdateItems(ISequentialOutStream *outSeqStream, UInt32 nu } else { - inShaStreamSpec->SetStream(fileInStream); + inShaStream->SetStream(fileInStream); CMyComPtr inSeekStream; fileInStream.QueryInterface(IID_IInStream, (void **)&inSeekStream); fileInStream.Release(); - inShaStreamSpec->Init(); + inShaStream->Init(); UInt64 offsetBlockSize = 0; /* if (useResourceCompression) @@ -1711,16 +1674,16 @@ Z7_COM7F_IMF(CHandler::UpdateItems(ISequentialOutStream *outSeqStream, UInt32 nu if (inSeekStream /* && !sortedHashes.IsEmpty() */) { - RINOK(copyCoder->Code(inShaStream, NULL, NULL, NULL, progress)) - size = copyCoderSpec->TotalSize; + RINOK(copyCoder.Interface()->Code(inShaStream, NULL, NULL, NULL, lps)) + size = copyCoder->TotalSize; if (size == 0) needWritePass = false; else { Byte hash[kHashSize]; - inShaStreamSpec->Final(hash); + inShaStream->Final(hash); - index = AddUniqHash(&streams.Front(), sortedHashes, hash, -1); + index = AddUniqHash(streams.ConstData(), sortedHashes, hash, -1); if (index != -1) { streams[index].RefCount++; @@ -1729,15 +1692,15 @@ Z7_COM7F_IMF(CHandler::UpdateItems(ISequentialOutStream *outSeqStream, UInt32 nu else { RINOK(InStream_SeekToBegin(inSeekStream)) - inShaStreamSpec->Init(); + inShaStream->Init(); } } } if (needWritePass) { - RINOK(copyCoder->Code(inShaStream, outStream, NULL, NULL, progress)) - size = copyCoderSpec->TotalSize; + RINOK(copyCoder.Interface()->Code(inShaStream, outStream, NULL, NULL, lps)) + size = copyCoder->TotalSize; } if (size != 0) @@ -1746,9 +1709,9 @@ Z7_COM7F_IMF(CHandler::UpdateItems(ISequentialOutStream *outSeqStream, UInt32 nu { Byte hash[kHashSize]; const UInt64 packSize = offsetBlockSize + size; - inShaStreamSpec->Final(hash); + inShaStream->Final(hash); - index = AddUniqHash(&streams.Front(), sortedHashes, hash, (int)streams.Size()); + index = AddUniqHash(streams.ConstData(), sortedHashes, hash, (int)streams.Size()); if (index != -1) { @@ -1805,8 +1768,8 @@ Z7_COM7F_IMF(CHandler::UpdateItems(ISequentialOutStream *outSeqStream, UInt32 nu RINOK(InStream_SeekSet(_volumes[1].Stream, s.Resource.Offset)) inStreamLimitedSpec->Init(s.Resource.PackSize); - RINOK(copyCoder->Code(inStreamLimited, outStream, NULL, NULL, progress)) - if (copyCoderSpec->TotalSize != s.Resource.PackSize) + RINOK(copyCoder.Interface()->Code(inStreamLimited, outStream, NULL, NULL, lps)) + if (copyCoder->TotalSize != s.Resource.PackSize) return E_FAIL; s.Resource.Offset = curPos; @@ -1874,7 +1837,7 @@ Z7_COM7F_IMF(CHandler::UpdateItems(ISequentialOutStream *outSeqStream, UInt32 nu Set32((Byte *)meta, (UInt32)pos) // size of security data } - db.Hashes = &streams.Front(); + db.Hashes = streams.ConstData(); db.WriteTree(tree, (Byte *)meta, pos); { diff --git a/CPP/7zip/Archive/Wim/WimIn.cpp b/CPP/7zip/Archive/Wim/WimIn.cpp index ff118d8..614755a 100644 --- a/CPP/7zip/Archive/Wim/WimIn.cpp +++ b/CPP/7zip/Archive/Wim/WimIn.cpp @@ -34,19 +34,19 @@ namespace NArchive { namespace NWim { -static int inline GetLog(UInt32 num) +static bool inline GetLog_val_min_dest(const UInt32 val, unsigned i, unsigned &dest) { - for (int i = 0; i < 32; i++) - if (((UInt32)1 << i) == num) - return i; - return -1; -} - - -CUnpacker::~CUnpacker() -{ - if (lzmsDecoder) - delete lzmsDecoder; + UInt32 v = (UInt32)1 << i; + for (; i < 32; i++) + { + if (v == val) + { + dest = i; + return true; + } + v += v; + } + return false; } @@ -64,25 +64,27 @@ HRESULT CUnpacker::UnpackChunk( } else if (method == NMethod::kLZX) { - if (!lzxDecoder) - { - lzxDecoderSpec = new NCompress::NLzx::CDecoder(true); - lzxDecoder = lzxDecoderSpec; - } + lzxDecoder.Create_if_Empty(); + lzxDecoder->Set_WimMode(true); } else if (method == NMethod::kLZMS) { - if (!lzmsDecoder) - lzmsDecoder = new NCompress::NLzms::CDecoder(); + lzmsDecoder.Create_if_Empty(); } else return E_NOTIMPL; const size_t chunkSize = (size_t)1 << chunkSizeBits; - - unpackBuf.EnsureCapacity(chunkSize); - if (!unpackBuf.Data) - return E_OUTOFMEMORY; + + { + const unsigned + kAdditionalOutputBufSize = MyMax(NCompress::NLzx:: + kAdditionalOutputBufSize, NCompress::NXpress:: + kAdditionalOutputBufSize); + unpackBuf.EnsureCapacity(chunkSize + kAdditionalOutputBufSize); + if (!unpackBuf.Data) + return E_OUTOFMEMORY; + } HRESULT res = S_FALSE; size_t unpackedSize = 0; @@ -95,30 +97,32 @@ HRESULT CUnpacker::UnpackChunk( } else if (inSize < chunkSize) { - packBuf.EnsureCapacity(chunkSize); + const unsigned kAdditionalInputSize = 32; + packBuf.EnsureCapacity(chunkSize + kAdditionalInputSize); if (!packBuf.Data) return E_OUTOFMEMORY; RINOK(ReadStream_FALSE(inStream, packBuf.Data, inSize)) + memset(packBuf.Data + inSize, 0xff, kAdditionalInputSize); TotalPacked += inSize; if (method == NMethod::kXPRESS) { - res = NCompress::NXpress::Decode(packBuf.Data, inSize, unpackBuf.Data, outSize); + res = NCompress::NXpress::Decode_WithExceedWrite(packBuf.Data, inSize, unpackBuf.Data, outSize); if (res == S_OK) unpackedSize = outSize; } else if (method == NMethod::kLZX) { - res = lzxDecoderSpec->SetExternalWindow(unpackBuf.Data, chunkSizeBits); + res = lzxDecoder->Set_ExternalWindow_DictBits(unpackBuf.Data, chunkSizeBits); if (res != S_OK) return E_NOTIMPL; - lzxDecoderSpec->KeepHistoryForNext = false; - lzxDecoderSpec->SetKeepHistory(false); - res = lzxDecoderSpec->Code(packBuf.Data, inSize, (UInt32)outSize); - unpackedSize = lzxDecoderSpec->GetUnpackSize(); - if (res == S_OK && !lzxDecoderSpec->WasBlockFinished()) + lzxDecoder->Set_KeepHistoryForNext(false); + lzxDecoder->Set_KeepHistory(false); + res = lzxDecoder->Code_WithExceedReadWrite(packBuf.Data, inSize, (UInt32)outSize); + unpackedSize = lzxDecoder->GetUnpackSize(); + if (res == S_OK && !lzxDecoder->WasBlockFinished()) res = S_FALSE; } else @@ -158,26 +162,21 @@ HRESULT CUnpacker::Unpack2( { if (!resource.IsCompressed() && !resource.IsSolid()) { - if (!copyCoder) - { - copyCoderSpec = new NCompress::CCopyCoder; - copyCoder = copyCoderSpec; - } + copyCoder.Create_if_Empty(); - CLimitedSequentialInStream *limitedStreamSpec = new CLimitedSequentialInStream(); - CMyComPtr limitedStream = limitedStreamSpec; - limitedStreamSpec->SetStream(inStream); + CMyComPtr2_Create limitedStream; + limitedStream->SetStream(inStream); RINOK(InStream_SeekSet(inStream, resource.Offset)) if (resource.PackSize != resource.UnpackSize) return S_FALSE; - limitedStreamSpec->Init(resource.PackSize); + limitedStream->Init(resource.PackSize); TotalPacked += resource.PackSize; - HRESULT res = copyCoder->Code(limitedStream, outStream, NULL, NULL, progress); + HRESULT res = copyCoder.Interface()->Code(limitedStream, outStream, NULL, NULL, progress); - if (res == S_OK && copyCoderSpec->TotalSize != resource.UnpackSize) + if (res == S_OK && copyCoder->TotalSize != resource.UnpackSize) res = S_FALSE; return res; } @@ -366,24 +365,13 @@ HRESULT CUnpacker::Unpack2( HRESULT CUnpacker::Unpack(IInStream *inStream, const CResource &resource, const CHeader &header, const CDatabase *db, ISequentialOutStream *outStream, ICompressProgressInfo *progress, Byte *digest) { - COutStreamWithSha1 *shaStreamSpec = NULL; - CMyComPtr shaStream; - + CMyComPtr2_Create shaStream; // outStream can be NULL, so we use COutStreamWithSha1 even if sha1 is not required - // if (digest) - { - shaStreamSpec = new COutStreamWithSha1(); - shaStream = shaStreamSpec; - shaStreamSpec->SetStream(outStream); - shaStreamSpec->Init(digest != NULL); - outStream = shaStream; - } - - HRESULT res = Unpack2(inStream, resource, header, db, outStream, progress); - + shaStream->SetStream(outStream); + shaStream->Init(digest != NULL); + const HRESULT res = Unpack2(inStream, resource, header, db, shaStream, progress); if (digest) - shaStreamSpec->Final(digest); - + shaStream->Final(digest); return res; } @@ -394,21 +382,16 @@ HRESULT CUnpacker::UnpackData(IInStream *inStream, CByteBuffer &buf, Byte *digest) { // if (resource.IsSolid()) return E_NOTIMPL; - UInt64 unpackSize64 = resource.UnpackSize; if (db) unpackSize64 = db->Get_UnpackSize_of_Resource(resource); - - size_t size = (size_t)unpackSize64; + const size_t size = (size_t)unpackSize64; if (size != unpackSize64) return E_OUTOFMEMORY; - buf.Alloc(size); - CBufPtrSeqOutStream *outStreamSpec = new CBufPtrSeqOutStream(); - CMyComPtr outStream = outStreamSpec; - outStreamSpec->Init((Byte *)buf, size); - + CMyComPtr2_Create outStream; + outStream->Init((Byte *)buf, size); return Unpack(inStream, resource, header, db, outStream, NULL, digest); } @@ -900,10 +883,8 @@ HRESULT CHeader::Parse(const Byte *p, UInt64 &phySize) ChunkSizeBits = kChunkSizeBits; if (ChunkSize != 0) { - const int log = GetLog(ChunkSize); - if (log < 12) + if (!GetLog_val_min_dest(ChunkSize, 12, ChunkSizeBits)) return S_FALSE; - ChunkSizeBits = (unsigned)log; } } @@ -1314,23 +1295,21 @@ HRESULT CDatabase::FillAndCheck(const CObjectVector &volumes) return S_FALSE; const UInt32 solidChunkSize = GetUi32(header + 8); - const int log = GetLog(solidChunkSize); - if (log < 8 || log > 31) + if (!GetLog_val_min_dest(solidChunkSize, 8, ss.ChunkSizeBits)) return S_FALSE; - ss.ChunkSizeBits = (unsigned)log; ss.Method = (Int32)GetUi32(header + 12); - UInt64 numChunks64 = (ss.UnpackSize + (((UInt32)1 << ss.ChunkSizeBits) - 1)) >> ss.ChunkSizeBits; - UInt64 sizesBufSize64 = 4 * numChunks64; + const UInt64 numChunks64 = (ss.UnpackSize + (((UInt32)1 << ss.ChunkSizeBits) - 1)) >> ss.ChunkSizeBits; + const UInt64 sizesBufSize64 = 4 * numChunks64; ss.HeadersSize = kSolidHeaderSize + sizesBufSize64; - size_t sizesBufSize = (size_t)sizesBufSize64; + const size_t sizesBufSize = (size_t)sizesBufSize64; if (sizesBufSize != sizesBufSize64) return E_OUTOFMEMORY; sizesBuf.AllocAtLeast(sizesBufSize); RINOK(ReadStream_FALSE(inStream, sizesBuf, sizesBufSize)) - size_t numChunks = (size_t)numChunks64; + const size_t numChunks = (size_t)numChunks64; ss.Chunks.Alloc(numChunks + 1); UInt64 offset = 0; @@ -1439,7 +1418,7 @@ HRESULT CDatabase::FillAndCheck(const CObjectVector &volumes) { { - const CStreamInfo *streams = &DataStreams.Front(); + const CStreamInfo *streams = DataStreams.ConstData(); if (IsOldVersion) { @@ -1481,7 +1460,7 @@ HRESULT CDatabase::FillAndCheck(const CObjectVector &volumes) hash += (item.IsAltStream ? 0x8 : 0x10); UInt32 id = GetUi32(hash); if (id != 0) - item.StreamIndex = FindId(&DataStreams.Front(), sortedByHash, id); + item.StreamIndex = FindId(DataStreams.ConstData(), sortedByHash, id); } } /* @@ -1495,7 +1474,7 @@ HRESULT CDatabase::FillAndCheck(const CObjectVector &volumes) hash += (item.IsAltStream ? 0x10 : 0x40); if (!IsEmptySha(hash)) { - item.StreamIndex = FindHash(&DataStreams.Front(), sortedByHash, hash); + item.StreamIndex = FindHash(DataStreams.ConstData(), sortedByHash, hash); } } } @@ -1518,7 +1497,7 @@ HRESULT CDatabase::FillAndCheck(const CObjectVector &volumes) for (i = 0; i < Items.Size(); i++) { - int streamIndex = Items[i].StreamIndex; + const int streamIndex = Items[i].StreamIndex; if (streamIndex >= 0) refCounts[streamIndex]++; } @@ -1773,13 +1752,12 @@ static bool ParseNumber32(const AString &s, UInt32 &res) static bool ParseTime(const CXmlItem &item, FILETIME &ft, const char *tag) { - int index = item.FindSubTag(tag); - if (index >= 0) + const CXmlItem *timeItem = item.FindSubTag_GetPtr(tag); + if (timeItem) { - const CXmlItem &timeItem = item.SubItems[index]; UInt32 low = 0, high = 0; - if (ParseNumber32(timeItem.GetSubStringForTag("LOWPART"), low) && - ParseNumber32(timeItem.GetSubStringForTag("HIGHPART"), high)) + if (ParseNumber32(timeItem->GetSubStringForTag("LOWPART"), low) && + ParseNumber32(timeItem->GetSubStringForTag("HIGHPART"), high)) { ft.dwLowDateTime = low; ft.dwHighDateTime = high; diff --git a/CPP/7zip/Archive/Wim/WimIn.h b/CPP/7zip/Archive/Wim/WimIn.h index 3de8456..71d0a3a 100644 --- a/CPP/7zip/Archive/Wim/WimIn.h +++ b/CPP/7zip/Archive/Wim/WimIn.h @@ -5,6 +5,7 @@ #include "../../../../C/Alloc.h" +#include "../../../Common/AutoPtr.h" #include "../../../Common/MyBuffer.h" #include "../../../Common/MyXml.h" @@ -583,27 +584,23 @@ struct CMidBuf { if (size > _size) { - ::MidFree(Data); + ::z7_AlignedFree(Data); _size = 0; - Data = (Byte *)::MidAlloc(size); + Data = (Byte *)::z7_AlignedAlloc(size); if (Data) _size = size; } } - ~CMidBuf() { ::MidFree(Data); } + ~CMidBuf() { ::z7_AlignedFree(Data); } }; class CUnpacker { - NCompress::CCopyCoder *copyCoderSpec; - CMyComPtr copyCoder; - - NCompress::NLzx::CDecoder *lzxDecoderSpec; - CMyComPtr lzxDecoder; - - NCompress::NLzms::CDecoder *lzmsDecoder; + CMyComPtr2 copyCoder; + CMyUniquePtr lzxDecoder; + CMyUniquePtr lzmsDecoder; CByteBuffer sizesBuf; @@ -637,7 +634,6 @@ class CUnpacker _unpackedChunkIndex(0), TotalPacked(0) {} - ~CUnpacker(); HRESULT Unpack( IInStream *inStream, diff --git a/CPP/7zip/Archive/XarHandler.cpp b/CPP/7zip/Archive/XarHandler.cpp index c03128f..5112a16 100644 --- a/CPP/7zip/Archive/XarHandler.cpp +++ b/CPP/7zip/Archive/XarHandler.cpp @@ -2,6 +2,7 @@ #include "StdAfx.h" +#include "../../../C/Sha256.h" #include "../../../C/CpuArch.h" #include "../../Common/ComTry.h" @@ -36,58 +37,205 @@ using namespace NWindows; namespace NArchive { namespace NXar { -static const size_t kXmlSizeMax = ((size_t )1 << 30) - (1 << 14); -static const size_t kXmlPackSizeMax = kXmlSizeMax; +Z7_CLASS_IMP_NOQIB_1( + CInStreamWithSha256 + , ISequentialInStream +) + CMyComPtr _stream; + CAlignedBuffer1 _sha; + UInt64 _size; + + CSha256 *Sha() { return (CSha256 *)(void *)(Byte *)_sha; } +public: + CInStreamWithSha256(): _sha(sizeof(CSha256)) {} + void SetStream(ISequentialInStream *stream) { _stream = stream; } + void Init() + { + _size = 0; + Sha256_Init(Sha()); + } + void ReleaseStream() { _stream.Release(); } + UInt64 GetSize() const { return _size; } + void Final(Byte *digest) { Sha256_Final(Sha(), digest); } +}; -/* -#define XAR_CKSUM_NONE 0 -#define XAR_CKSUM_SHA1 1 -#define XAR_CKSUM_MD5 2 +Z7_COM7F_IMF(CInStreamWithSha256::Read(void *data, UInt32 size, UInt32 *processedSize)) +{ + UInt32 realProcessedSize; + const HRESULT result = _stream->Read(data, size, &realProcessedSize); + _size += realProcessedSize; + Sha256_Update(Sha(), (const Byte *)data, realProcessedSize); + if (processedSize) + *processedSize = realProcessedSize; + return result; +} + + +Z7_CLASS_IMP_NOQIB_1( + COutStreamWithSha256 + , ISequentialOutStream +) + // bool _calculate; + CMyComPtr _stream; + CAlignedBuffer1 _sha; + UInt64 _size; + + CSha256 *Sha() { return (CSha256 *)(void *)(Byte *)_sha; } +public: + COutStreamWithSha256(): _sha(sizeof(CSha256)) {} + void SetStream(ISequentialOutStream *stream) { _stream = stream; } + void ReleaseStream() { _stream.Release(); } + void Init(/* bool calculate = true */ ) + { + // _calculate = calculate; + _size = 0; + Sha256_Init(Sha()); + } + void InitSha256() { Sha256_Init(Sha()); } + UInt64 GetSize() const { return _size; } + void Final(Byte *digest) { Sha256_Final(Sha(), digest); } +}; + +Z7_COM7F_IMF(COutStreamWithSha256::Write(const void *data, UInt32 size, UInt32 *processedSize)) +{ + HRESULT result = S_OK; + if (_stream) + result = _stream->Write(data, size, &size); + // if (_calculate) + Sha256_Update(Sha(), (const Byte *)data, size); + _size += size; + if (processedSize) + *processedSize = size; + return result; +} + +// we limit supported xml sizes: +// ((size_t)1 << (sizeof(size_t) / 2 + 28)) - (1u << 14); +static const size_t kXmlSizeMax = ((size_t)1 << 30) - (1u << 14); +static const size_t kXmlPackSizeMax = kXmlSizeMax; -static const char * const k_ChecksumAlgos[] = +#define XAR_CKSUM_NONE 0 +#define XAR_CKSUM_SHA1 1 +#define XAR_CKSUM_MD5 2 +#define XAR_CKSUM_SHA256 3 +#define XAR_CKSUM_SHA512 4 +// #define XAR_CKSUM_OTHER 3 +// fork version of xar can use (3) as special case, +// where name of hash is stored as string at the end of header +// we do not support such hash still. + +static const char * const k_ChecksumNames[] = { - "None" - , "SHA-1" + "NONE" + , "SHA1" , "MD5" + , "SHA256" + , "SHA512" }; -*/ -#define METHOD_NAME_ZLIB "zlib" +static unsigned GetHashSize(int algo) +{ + if (algo <= XAR_CKSUM_NONE || algo > XAR_CKSUM_SHA512) + return 0; + if (algo == XAR_CKSUM_SHA1) + return SHA1_DIGEST_SIZE; + return (16u >> XAR_CKSUM_MD5) << algo; +} + +#define METHOD_NAME_ZLIB "zlib" + +static int Find_ChecksumId_for_Name(const AString &style) +{ + for (unsigned i = 0; i < Z7_ARRAY_SIZE(k_ChecksumNames); i++) + { + // old xars used upper case in "style" + // new xars use lower case in "style" + if (style.IsEqualTo_Ascii_NoCase(k_ChecksumNames[i])) + return (int)i; + } + return -1; +} + + +struct CCheckSum +{ + int AlgoNumber; + bool Error; + CByteBuffer Data; + AString Style; + + CCheckSum(): AlgoNumber(-1), Error(false) {} + void AddNameToString(AString &s) const; +}; + +void CCheckSum::AddNameToString(AString &s) const +{ + if (Style.IsEmpty()) + s.Add_OptSpaced("NO-CHECKSUM"); + else + { + s.Add_OptSpaced(Style); + if (Error) + s += "-ERROR"; + } +} struct CFile { - AString Name; - AString Method; + bool IsDir; + bool Is_SymLink; + bool HasData; + bool Mode_Defined; + bool INode_Defined; + bool UserId_Defined; + bool GroupId_Defined; + // bool Device_Defined; + bool Id_Defined; + + int Parent; + UInt32 Mode; + UInt64 Size; UInt64 PackSize; UInt64 Offset; - - UInt64 CTime; UInt64 MTime; + UInt64 CTime; UInt64 ATime; - UInt32 Mode; + UInt64 INode; + UInt64 UserId; + UInt64 GroupId; + // UInt64 Device; + AString Name; + AString Method; AString User; AString Group; + // AString Id; + AString Type; + AString Link; + // AString LinkType; + // AString LinkFrom; - bool IsDir; - bool HasData; - bool ModeDefined; - bool Sha1IsDefined; - // bool packSha1IsDefined; - - Byte Sha1[SHA1_DIGEST_SIZE]; - // Byte packSha1[SHA1_DIGEST_SIZE]; - - int Parent; - - CFile(): + UInt64 Id; + CCheckSum extracted_checksum; + CCheckSum archived_checksum; + + CFile(int parent): + IsDir(false), + Is_SymLink(false), + HasData(false), + Mode_Defined(false), + INode_Defined(false), + UserId_Defined(false), + GroupId_Defined(false), + // Device_Defined(false), + Id_Defined(false), + Parent(parent), + Mode(0), Size(0), PackSize(0), Offset(0), - CTime(0), MTime(0), ATime(0), Mode(0), - IsDir(false), HasData(false), ModeDefined(false), Sha1IsDefined(false), - /* packSha1IsDefined(false), */ - Parent(-1) + MTime(0), CTime(0), ATime(0), + INode(0) {} bool IsCopyMethod() const @@ -97,36 +245,46 @@ struct CFile void UpdateTotalPackSize(UInt64 &totalSize) const { - UInt64 t = Offset + PackSize; + const UInt64 t = Offset + PackSize; + if (t >= Offset) if (totalSize < t) - totalSize = t; + totalSize = t; } }; -Z7_CLASS_IMP_CHandler_IInArchive_1( +Z7_CLASS_IMP_CHandler_IInArchive_2( + IArchiveGetRawProps, IInArchiveGetStream ) - UInt64 _dataStartPos; - CMyComPtr _inStream; - CByteArr _xml; - size_t _xmlLen; + bool _is_pkg; + bool _toc_CrcError; CObjectVector _files; - // UInt32 _checkSumAlgo; + CMyComPtr _inStream; + UInt64 _dataStartPos; UInt64 _phySize; + CAlignedBuffer _xmlBuf; + size_t _xmlLen; + // UInt64 CreationTime; + AString CreationTime_String; + UInt32 _checkSumAlgo; Int32 _mainSubfile; - bool _is_pkg; HRESULT Open2(IInStream *stream); - HRESULT Extract(IInStream *stream); }; + static const Byte kArcProps[] = { kpidSubType, - kpidHeadersSize + // kpidHeadersSize, + kpidMethod, + kpidCTime }; +// #define kpidLinkType 250 +// #define kpidLinkFrom 251 + static const Byte kProps[] = { kpidPath, @@ -136,19 +294,25 @@ static const Byte kProps[] = kpidCTime, kpidATime, kpidPosixAttrib, + kpidType, kpidUser, kpidGroup, - kpidMethod + kpidUserId, + kpidGroupId, + kpidINode, + // kpidDeviceMajor, + // kpidDeviceMinor, + kpidSymLink, + // kpidLinkType, + // kpidLinkFrom, + kpidMethod, + kpidId, + kpidOffset }; IMP_IInArchive_Props IMP_IInArchive_ArcProps -#define PARSE_NUM(_num_, _dest_) \ - { const char *end; _dest_ = ConvertStringToUInt32(p, &end); \ - if ((unsigned)(end - p) != _num_) return 0; \ - p += _num_ + 1; } - static bool ParseUInt64(const CXmlItem &item, const char *name, UInt64 &res) { const AString s (item.GetSubStringForTag(name)); @@ -159,14 +323,26 @@ static bool ParseUInt64(const CXmlItem &item, const char *name, UInt64 &res) return *end == 0; } -static UInt64 ParseTime(const CXmlItem &item, const char *name) + +#define PARSE_NUM(_num_, _dest_) \ + { const char *end; _dest_ = ConvertStringToUInt32(p, &end); \ + if ((unsigned)(end - p) != _num_) return 0; \ + p += _num_ + 1; } + +static UInt64 ParseTime(const CXmlItem &item, const char *name /* , bool z_isRequired */ ) { const AString s (item.GetSubStringForTag(name)); - if (s.Len() < 20) + if (s.Len() < 20 /* (z_isRequired ? 20u : 19u) */) return 0; const char *p = s; - if (p[ 4] != '-' || p[ 7] != '-' || p[10] != 'T' || - p[13] != ':' || p[16] != ':' || p[19] != 'Z') + if (p[ 4] != '-' || + p[ 7] != '-' || + p[10] != 'T' || + p[13] != ':' || + p[16] != ':') + return 0; + // if (z_isRequired) + if (p[19] != 'Z') return 0; UInt32 year, month, day, hour, min, sec; PARSE_NUM(4, year) @@ -175,176 +351,261 @@ static UInt64 ParseTime(const CXmlItem &item, const char *name) PARSE_NUM(2, hour) PARSE_NUM(2, min) PARSE_NUM(2, sec) - UInt64 numSecs; if (!NTime::GetSecondsSince1601(year, month, day, hour, min, sec, numSecs)) return 0; return numSecs * 10000000; } -static int HexToByte(char c) -{ - if (c >= '0' && c <= '9') return c - '0'; - if (c >= 'A' && c <= 'F') return c - 'A' + 10; - if (c >= 'a' && c <= 'f') return c - 'a' + 10; - return -1; -} -static bool ParseSha1(const CXmlItem &item, const char *name, Byte *digest) +static void ParseChecksum(const CXmlItem &item, const char *name, CCheckSum &checksum) { - const int index = item.FindSubTag(name); - if (index < 0) - return false; - const CXmlItem &checkItem = item.SubItems[index]; - const AString style (checkItem.GetPropVal("style")); - if (style == "SHA1") + const CXmlItem *checkItem = item.FindSubTag_GetPtr(name); + if (!checkItem) + return; // false; + checksum.Style = checkItem->GetPropVal("style"); + const AString s (checkItem->GetSubString()); + if ((s.Len() & 1) == 0 && s.Len() <= (2u << 7)) // 1024-bit max { - const AString s (checkItem.GetSubString()); - if (s.Len() != SHA1_DIGEST_SIZE * 2) - return false; - for (unsigned i = 0; i < s.Len(); i += 2) + const size_t size = s.Len() / 2; + CByteBuffer temp(size); + if ((size_t)(ParseHexString(s, temp) - temp) == size) { - const int b0 = HexToByte(s[i]); - const int b1 = HexToByte(s[i + 1]); - if (b0 < 0 || b1 < 0) - return false; - digest[i / 2] = (Byte)((b0 << 4) | b1); + checksum.Data = temp; + const int index = Find_ChecksumId_for_Name(checksum.Style); + if (index >= 0 && checksum.Data.Size() == GetHashSize(index)) + { + checksum.AlgoNumber = index; + return; + } } - return true; } - return false; + checksum.Error = true; } -static bool AddItem(const CXmlItem &item, CObjectVector &files, int parent) + +static bool AddItem(const CXmlItem &item, CObjectVector &files, int parent, int level) { if (!item.IsTag) return true; + if (level >= 1024) + return false; if (item.Name == "file") { - CFile file; - file.Parent = parent; + CFile file(parent); parent = (int)files.Size(); + { + const AString id = item.GetPropVal("id"); + const char *end; + file.Id = ConvertStringToUInt64(id, &end); + if (*end == 0) + file.Id_Defined = true; + } file.Name = item.GetSubStringForTag("name"); - const AString type (item.GetSubStringForTag("type")); - if (type == "directory") - file.IsDir = true; - else if (type == "file") - file.IsDir = false; - else - return false; + z7_xml_DecodeString(file.Name); + { + const CXmlItem *typeItem = item.FindSubTag_GetPtr("type"); + if (typeItem) + { + file.Type = typeItem->GetSubString(); + // file.LinkFrom = typeItem->GetPropVal("link"); + if (file.Type == "directory") + file.IsDir = true; + else + { + // file.IsDir = false; + /* + else if (file.Type == "file") + {} + else if (file.Type == "hardlink") + {} + else + */ + if (file.Type == "symlink") + file.Is_SymLink = true; + // file.IsDir = false; + } + } + } + { + const CXmlItem *linkItem = item.FindSubTag_GetPtr("link"); + if (linkItem) + { + // file.LinkType = linkItem->GetPropVal("type"); + file.Link = linkItem->GetSubString(); + z7_xml_DecodeString(file.Link); + } + } - int dataIndex = item.FindSubTag("data"); - if (dataIndex >= 0 && !file.IsDir) + const CXmlItem *dataItem = item.FindSubTag_GetPtr("data"); + if (dataItem && !file.IsDir) { file.HasData = true; - const CXmlItem &dataItem = item.SubItems[dataIndex]; - if (!ParseUInt64(dataItem, "size", file.Size)) + if (!ParseUInt64(*dataItem, "size", file.Size)) return false; - if (!ParseUInt64(dataItem, "length", file.PackSize)) + if (!ParseUInt64(*dataItem, "length", file.PackSize)) return false; - if (!ParseUInt64(dataItem, "offset", file.Offset)) + if (!ParseUInt64(*dataItem, "offset", file.Offset)) return false; - file.Sha1IsDefined = ParseSha1(dataItem, "extracted-checksum", file.Sha1); - // file.packSha1IsDefined = ParseSha1(dataItem, "archived-checksum", file.packSha1); - int encodingIndex = dataItem.FindSubTag("encoding"); - if (encodingIndex >= 0) + ParseChecksum(*dataItem, "extracted-checksum", file.extracted_checksum); + ParseChecksum(*dataItem, "archived-checksum", file.archived_checksum); + const CXmlItem *encodingItem = dataItem->FindSubTag_GetPtr("encoding"); + if (encodingItem) { - const CXmlItem &encodingItem = dataItem.SubItems[encodingIndex]; - if (encodingItem.IsTag) + AString s (encodingItem->GetPropVal("style")); + if (!s.IsEmpty()) { - AString s (encodingItem.GetPropVal("style")); - if (!s.IsEmpty()) + const AString appl ("application/"); + if (s.IsPrefixedBy(appl)) { - const AString appl ("application/"); - if (s.IsPrefixedBy(appl)) + s.DeleteFrontal(appl.Len()); + const AString xx ("x-"); + if (s.IsPrefixedBy(xx)) { - s.DeleteFrontal(appl.Len()); - const AString xx ("x-"); - if (s.IsPrefixedBy(xx)) - { - s.DeleteFrontal(xx.Len()); - if (s == "gzip") - s = METHOD_NAME_ZLIB; - } + s.DeleteFrontal(xx.Len()); + if (s == "gzip") + s = METHOD_NAME_ZLIB; } - file.Method = s; } + file.Method = s; } } } + file.INode_Defined = ParseUInt64(item, "inode", file.INode); + file.UserId_Defined = ParseUInt64(item, "uid", file.UserId); + file.GroupId_Defined = ParseUInt64(item, "gid", file.GroupId); + // file.Device_Defined = ParseUInt64(item, "deviceno", file.Device); + file.MTime = ParseTime(item, "mtime"); // z_IsRequied = true file.CTime = ParseTime(item, "ctime"); - file.MTime = ParseTime(item, "mtime"); file.ATime = ParseTime(item, "atime"); - { const AString s (item.GetSubStringForTag("mode")); if (s[0] == '0') { const char *end; file.Mode = ConvertOctStringToUInt32(s, &end); - file.ModeDefined = (*end == 0); + file.Mode_Defined = (*end == 0); } } - file.User = item.GetSubStringForTag("user"); file.Group = item.GetSubStringForTag("group"); files.Add(file); } + FOR_VECTOR (i, item.SubItems) - if (!AddItem(item.SubItems[i], files, parent)) + if (!AddItem(item.SubItems[i], files, parent, level + 1)) return false; return true; } -HRESULT CHandler::Open2(IInStream *stream) + + +struct CInStreamWithHash { - const UInt32 kHeaderSize = 0x1C; - Byte buf[kHeaderSize]; - RINOK(ReadStream_FALSE(stream, buf, kHeaderSize)) - UInt32 size = Get16(buf + 4); - // UInt32 ver = Get16(buf + 6); // == 1 - if (Get32(buf) != 0x78617221 || size != kHeaderSize) - return S_FALSE; + CMyComPtr2_Create inStreamSha1; + CMyComPtr2_Create inStreamSha256; + CMyComPtr2_Create inStreamLim; - UInt64 packSize = Get64(buf + 8); - UInt64 unpackSize = Get64(buf + 0x10); + void SetStreamAndInit(ISequentialInStream *stream, int algo); + bool CheckHash(int algo, const Byte *digest_from_arc) const; +}; - // _checkSumAlgo = Get32(buf + 0x18); +void CInStreamWithHash::SetStreamAndInit(ISequentialInStream *stream, int algo) +{ + if (algo == XAR_CKSUM_SHA1) + { + inStreamSha1->SetStream(stream); + inStreamSha1->Init(); + stream = inStreamSha1; + } + else if (algo == XAR_CKSUM_SHA256) + { + inStreamSha256->SetStream(stream); + inStreamSha256->Init(); + stream = inStreamSha256; + } + inStreamLim->SetStream(stream); +} + +bool CInStreamWithHash::CheckHash(int algo, const Byte *digest_from_arc) const +{ + if (algo == XAR_CKSUM_SHA1) + { + Byte digest[SHA1_DIGEST_SIZE]; + inStreamSha1->Final(digest); + if (memcmp(digest, digest_from_arc, sizeof(digest)) != 0) + return false; + } + else if (algo == XAR_CKSUM_SHA256) + { + Byte digest[SHA256_DIGEST_SIZE]; + inStreamSha256->Final(digest); + if (memcmp(digest, digest_from_arc, sizeof(digest)) != 0) + return false; + } + return true; +} + + +HRESULT CHandler::Open2(IInStream *stream) +{ + const unsigned kHeaderSize = 28; + UInt32 buf32[kHeaderSize / sizeof(UInt32)]; + RINOK(ReadStream_FALSE(stream, buf32, kHeaderSize)) + const unsigned headerSize = Get16((const Byte *)(const void *)buf32 + 4); + // xar library now writes 1 to version field. + // some old xars could have version == 0 ? + // specification allows (headerSize != 28), + // but we don't expect big value in (headerSize). + // so we restrict (headerSize) with 64 bytes to reduce false open. + const unsigned kHeaderSize_MAX = 64; + if (Get32(buf32) != 0x78617221 // signature: "xar!" + || headerSize < kHeaderSize + || headerSize > kHeaderSize_MAX + || Get16((const Byte *)(const void *)buf32 + 6) > 1 // version + ) + return S_FALSE; + _checkSumAlgo = Get32(buf32 + 6); + const UInt64 packSize = Get64(buf32 + 2); + const UInt64 unpackSize = Get64(buf32 + 4); if (packSize >= kXmlPackSizeMax || unpackSize >= kXmlSizeMax) return S_FALSE; - - _dataStartPos = kHeaderSize + packSize; + /* some xar archives can have padding bytes at offset 28, + or checksum algorithm name at offset 28 (in xar fork, if cksum_alg==3) + But we didn't see such xar archives. + */ + if (headerSize != kHeaderSize) + { + RINOK(InStream_SeekSet(stream, headerSize)) + } + _dataStartPos = headerSize + packSize; _phySize = _dataStartPos; - _xml.Alloc((size_t)unpackSize + 1); + _xmlBuf.Alloc((size_t)unpackSize + 1); + if (!_xmlBuf.IsAllocated()) + return E_OUTOFMEMORY; _xmlLen = (size_t)unpackSize; - NCompress::NZlib::CDecoder *zlibCoderSpec = new NCompress::NZlib::CDecoder(); - CMyComPtr zlibCoder = zlibCoderSpec; - - CLimitedSequentialInStream *inStreamLimSpec = new CLimitedSequentialInStream; - CMyComPtr inStreamLim(inStreamLimSpec); - inStreamLimSpec->SetStream(stream); - inStreamLimSpec->Init(packSize); - - CBufPtrSeqOutStream *outStreamLimSpec = new CBufPtrSeqOutStream; - CMyComPtr outStreamLim(outStreamLimSpec); - outStreamLimSpec->Init(_xml, (size_t)unpackSize); - - RINOK(zlibCoder->Code(inStreamLim, outStreamLim, NULL, NULL, NULL)) - - if (outStreamLimSpec->GetPos() != (size_t)unpackSize) + CInStreamWithHash hashStream; + { + CMyComPtr2_Create zlibCoder; + hashStream.SetStreamAndInit(stream, (int)(unsigned)_checkSumAlgo); + hashStream.inStreamLim->Init(packSize); + CMyComPtr2_Create outStreamLim; + outStreamLim->Init(_xmlBuf, (size_t)unpackSize); + RINOK(zlibCoder.Interface()->Code(hashStream.inStreamLim, outStreamLim, NULL, &unpackSize, NULL)) + if (outStreamLim->GetPos() != (size_t)unpackSize) + return S_FALSE; + } + _xmlBuf[(size_t)unpackSize] = 0; + if (strlen((const char *)(const Byte *)_xmlBuf) != (size_t)unpackSize) return S_FALSE; - - _xml[(size_t)unpackSize] = 0; - if (strlen((const char *)(const Byte *)_xml) != unpackSize) return S_FALSE; - CXml xml; - if (!xml.Parse((const char *)(const Byte *)_xml)) + if (!xml.Parse((const char *)(const Byte *)_xmlBuf)) return S_FALSE; if (!xml.Root.IsTagged("xar") || xml.Root.SubItems.Size() != 1) @@ -352,7 +613,40 @@ HRESULT CHandler::Open2(IInStream *stream) const CXmlItem &toc = xml.Root.SubItems[0]; if (!toc.IsTagged("toc")) return S_FALSE; - if (!AddItem(toc, _files, -1)) + + // CreationTime = ParseTime(toc, "creation-time", false); // z_IsRequied + CreationTime_String = toc.GetSubStringForTag("creation-time"); + { + // we suppose that offset of checksum is always 0; + // but [TOC].xml contains exact offset value in block. + const UInt64 offset = 0; + const unsigned hashSize = GetHashSize((int)(unsigned)_checkSumAlgo); + if (hashSize) + { + /* + const CXmlItem *csItem = toc.FindSubTag_GetPtr("checksum"); + if (csItem) + { + const int checkSumAlgo2 = Find_ChecksumId_for_Name(csItem->GetPropVal("style")); + UInt64 csSize, csOffset; + if (ParseUInt64(*csItem, "size", csSize) && + ParseUInt64(*csItem, "offset", csOffset) && + csSize == hashSize && + (unsigned)checkSumAlgo2 == _checkSumAlgo) + offset = csOffset; + } + */ + CByteBuffer digest_from_arc(hashSize); + RINOK(InStream_SeekSet(stream, _dataStartPos + offset)) + RINOK(ReadStream_FALSE(stream, digest_from_arc, hashSize)) + if (!hashStream.CheckHash((int)(unsigned)_checkSumAlgo, digest_from_arc)) + _toc_CrcError = true; + } + } + + if (!AddItem(toc, _files, + -1, // parent + 0)) // level return S_FALSE; UInt64 totalPackSize = 0; @@ -362,19 +656,25 @@ HRESULT CHandler::Open2(IInStream *stream) { const CFile &file = _files[i]; file.UpdateTotalPackSize(totalPackSize); - if (file.Name == "Payload" || file.Name == "Content") + if (file.Parent == -1) { - _mainSubfile = (Int32)(int)i; - numMainFiles++; + if (file.Name == "Payload" || file.Name == "Content") + { + _mainSubfile = (Int32)(int)i; + numMainFiles++; + } + else if (file.Name == "PackageInfo") + _is_pkg = true; } - else if (file.Name == "PackageInfo") - _is_pkg = true; } if (numMainFiles > 1) _mainSubfile = -1; - - _phySize = _dataStartPos + totalPackSize; + + const UInt64 k_PhySizeLim = (UInt64)1 << 62; + _phySize = (totalPackSize > k_PhySizeLim - _dataStartPos) ? + k_PhySizeLim : + _dataStartPos + totalPackSize; return S_OK; } @@ -386,8 +686,7 @@ Z7_COM7F_IMF(CHandler::Open(IInStream *stream, COM_TRY_BEGIN { Close(); - if (Open2(stream) != S_OK) - return S_FALSE; + RINOK(Open2(stream)) _inStream = stream; } return S_OK; @@ -397,21 +696,26 @@ Z7_COM7F_IMF(CHandler::Open(IInStream *stream, Z7_COM7F_IMF(CHandler::Close()) { _phySize = 0; + _dataStartPos = 0; _inStream.Release(); _files.Clear(); _xmlLen = 0; - _xml.Free(); + _xmlBuf.Free(); _mainSubfile = -1; _is_pkg = false; + _toc_CrcError = false; + _checkSumAlgo = 0; + // CreationTime = 0; + CreationTime_String.Empty(); return S_OK; } Z7_COM7F_IMF(CHandler::GetNumberOfItems(UInt32 *numItems)) { *numItems = _files.Size() - #ifdef XAR_SHOW_RAW +#ifdef XAR_SHOW_RAW + 1 - #endif +#endif ; return S_OK; } @@ -443,40 +747,82 @@ Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) NCOM::CPropVariant prop; switch (propID) { - case kpidHeadersSize: prop = _dataStartPos; break; + // case kpidHeadersSize: prop = _dataStartPos; break; case kpidPhySize: prop = _phySize; break; case kpidMainSubfile: if (_mainSubfile >= 0) prop = (UInt32)_mainSubfile; break; case kpidSubType: if (_is_pkg) prop = "pkg"; break; case kpidExtension: prop = _is_pkg ? "pkg" : "xar"; break; + case kpidCTime: + { + // it's local time. We can transfer it to UTC time, if we use FILETIME. + // TimeToProp(CreationTime, prop); break; + if (!CreationTime_String.IsEmpty()) + prop = CreationTime_String; + break; + } + case kpidMethod: + { + AString s; + if (_checkSumAlgo < Z7_ARRAY_SIZE(k_ChecksumNames)) + s = k_ChecksumNames[_checkSumAlgo]; + else + { + s += "Checksum"; + s.Add_UInt32(_checkSumAlgo); + } + prop = s; + break; + } + case kpidWarningFlags: + { + UInt32 v = 0; + if (_toc_CrcError) v |= kpv_ErrorFlags_CrcError; + prop = v; + break; + } + case kpidINode: prop = true; break; + case kpidIsTree: prop = true; break; } prop.Detach(value); return S_OK; COM_TRY_END } + +/* +inline UInt32 MY_dev_major(UInt64 dev) +{ + return ((UInt32)(dev >> 8) & (UInt32)0xfff) | ((UInt32)(dev >> 32) & ~(UInt32)0xfff); +} +inline UInt32 MY_dev_minor(UInt64 dev) +{ + return ((UInt32)(dev) & 0xff) | ((UInt32)(dev >> 12) & ~(UInt32)0xff); +} +*/ + Z7_COM7F_IMF(CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *value)) { COM_TRY_BEGIN NCOM::CPropVariant prop; - #ifdef XAR_SHOW_RAW - if (index == _files.Size()) +#ifdef XAR_SHOW_RAW + if (index >= _files.Size()) { switch (propID) { - case kpidPath: prop = "[TOC].xml"; break; + case kpidName: + case kpidPath: + prop = "[TOC].xml"; break; case kpidSize: case kpidPackSize: prop = (UInt64)_xmlLen; break; } } else - #endif +#endif { const CFile &item = _files[index]; switch (propID) { - case kpidMethod: Utf8StringToProp(item.Method, prop); break; - case kpidPath: { AString path; @@ -486,37 +832,98 @@ Z7_COM7F_IMF(CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val const CFile &item2 = _files[cur]; if (!path.IsEmpty()) path.InsertAtFront(CHAR_PATH_SEPARATOR); +// #define XAR_EMPTY_NAME_REPLACEMENT "[]" if (item2.Name.IsEmpty()) - path.Insert(0, "unknown"); + { + AString s('['); + s.Add_UInt32(cur); + s.Add_Char(']'); + path.Insert(0, s); + } else path.Insert(0, item2.Name); - cur = (unsigned)item2.Parent; if (item2.Parent < 0) break; + cur = (unsigned)item2.Parent; } - Utf8StringToProp(path, prop); break; } - + + case kpidName: + { + if (item.Name.IsEmpty()) + { + AString s('['); + s.Add_UInt32(index); + s.Add_Char(']'); + prop = s; + } + else + Utf8StringToProp(item.Name, prop); + break; + } + case kpidIsDir: prop = item.IsDir; break; - case kpidSize: if (!item.IsDir) prop = item.Size; break; - case kpidPackSize: if (!item.IsDir) prop = item.PackSize; break; + case kpidSize: if (item.HasData && !item.IsDir) prop = item.Size; break; + case kpidPackSize: if (item.HasData && !item.IsDir) prop = item.PackSize; break; + + case kpidMethod: + { + if (item.HasData) + { + AString s = item.Method; + item.extracted_checksum.AddNameToString(s); + item.archived_checksum.AddNameToString(s); + Utf8StringToProp(s, prop); + } + break; + } + case kpidMTime: TimeToProp(item.MTime, prop); break; case kpidCTime: TimeToProp(item.CTime, prop); break; case kpidATime: TimeToProp(item.ATime, prop); break; + case kpidPosixAttrib: - if (item.ModeDefined) + if (item.Mode_Defined) { UInt32 mode = item.Mode; if ((mode & MY_LIN_S_IFMT) == 0) - mode |= (item.IsDir ? MY_LIN_S_IFDIR : MY_LIN_S_IFREG); + mode |= ( + item.Is_SymLink ? MY_LIN_S_IFLNK : + item.IsDir ? MY_LIN_S_IFDIR : + MY_LIN_S_IFREG); prop = mode; } break; - case kpidUser: Utf8StringToProp(item.User, prop); break; + + case kpidType: Utf8StringToProp(item.Type, prop); break; + case kpidUser: Utf8StringToProp(item.User, prop); break; case kpidGroup: Utf8StringToProp(item.Group, prop); break; + case kpidSymLink: if (item.Is_SymLink) Utf8StringToProp(item.Link, prop); break; + + case kpidUserId: if (item.UserId_Defined) prop = item.UserId; break; + case kpidGroupId: if (item.GroupId_Defined) prop = item.GroupId; break; + case kpidINode: if (item.INode_Defined) prop = item.INode; break; + case kpidId: if (item.Id_Defined) prop = item.Id; break; + // Utf8StringToProp(item.Id, prop); + /* + case kpidDeviceMajor: if (item.Device_Defined) prop = (UInt32)MY_dev_major(item.Device); break; + case kpidDeviceMinor: if (item.Device_Defined) prop = (UInt32)MY_dev_minor(item.Device); break; + case kpidLinkType: + if (!item.LinkType.IsEmpty()) + Utf8StringToProp(item.LinkType, prop); + break; + case kpidLinkFrom: + if (!item.LinkFrom.IsEmpty()) + Utf8StringToProp(item.LinkFrom, prop); + break; + */ + case kpidOffset: + if (item.HasData) + prop = _dataStartPos + item.Offset; + break; } } prop.Detach(value); @@ -524,75 +931,170 @@ Z7_COM7F_IMF(CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val COM_TRY_END } + +// for debug: +// #define Z7_XAR_SHOW_CHECKSUM_PACK + +#ifdef Z7_XAR_SHOW_CHECKSUM_PACK +enum +{ + kpidChecksumPack = kpidUserDefined +}; +#endif + +static const Byte kRawProps[] = +{ + kpidChecksum +#ifdef Z7_XAR_SHOW_CHECKSUM_PACK + , kpidCRC // instead of kpidUserDefined / kpidCRC +#endif +}; + +Z7_COM7F_IMF(CHandler::GetNumRawProps(UInt32 *numProps)) +{ + *numProps = Z7_ARRAY_SIZE(kRawProps); + return S_OK; +} + +Z7_COM7F_IMF(CHandler::GetRawPropInfo(UInt32 index, BSTR *name, PROPID *propID)) +{ + *propID = kRawProps[index]; + *name = NULL; + +#ifdef Z7_XAR_SHOW_CHECKSUM_PACK + if (index != 0) + { + *propID = kpidChecksumPack; + *name = NWindows::NCOM::AllocBstrFromAscii("archived-checksum"); + } +#endif + return S_OK; +} + +Z7_COM7F_IMF(CHandler::GetParent(UInt32 index, UInt32 *parent, UInt32 *parentType)) +{ + *parentType = NParentType::kDir; + *parent = (UInt32)(Int32)-1; +#ifdef XAR_SHOW_RAW + if (index >= _files.Size()) + return S_OK; +#endif + { + const CFile &item = _files[index]; + *parent = (UInt32)(Int32)item.Parent; + } + return S_OK; +} + + +Z7_COM7F_IMF(CHandler::GetRawProp(UInt32 index, PROPID propID, const void **data, UInt32 *dataSize, UInt32 *propType)) +{ + *data = NULL; + *dataSize = 0; + *propType = 0; + + // COM_TRY_BEGIN + NCOM::CPropVariant prop; + + if (propID == kpidChecksum) + { +#ifdef XAR_SHOW_RAW + if (index >= _files.Size()) + { + // case kpidPath: prop = "[TOC].xml"; break; + } + else +#endif + { + const CFile &item = _files[index]; + const size_t size = item.extracted_checksum.Data.Size(); + if (size != 0) + { + *dataSize = (UInt32)size; + *propType = NPropDataType::kRaw; + *data = item.extracted_checksum.Data; + } + } + } + +#ifdef Z7_XAR_SHOW_CHECKSUM_PACK + if (propID == kpidChecksumPack) + { +#ifdef XAR_SHOW_RAW + if (index >= _files.Size()) + { + // we can show digest check sum here + } + else +#endif + { + const CFile &item = _files[index]; + const size_t size = (UInt32)item.archived_checksum.Data.Size(); + if (size != 0) + { + *dataSize = (UInt32)size; + *propType = NPropDataType::kRaw; + *data = item.archived_checksum.Data; + } + } + } +#endif + return S_OK; +} + + + + Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, Int32 testMode, IArchiveExtractCallback *extractCallback)) { COM_TRY_BEGIN const bool allFilesMode = (numItems == (UInt32)(Int32)-1); if (allFilesMode) - numItems = _files.Size(); + numItems = _files.Size() +#ifdef XAR_SHOW_RAW + + 1 +#endif + ; if (numItems == 0) return S_OK; UInt64 totalSize = 0; UInt32 i; for (i = 0; i < numItems; i++) { - UInt32 index = (allFilesMode ? i : indices[i]); - #ifdef XAR_SHOW_RAW - if (index == _files.Size()) + const UInt32 index = allFilesMode ? i : indices[i]; +#ifdef XAR_SHOW_RAW + if (index >= _files.Size()) totalSize += _xmlLen; else - #endif +#endif totalSize += _files[index].Size; } - extractCallback->SetTotal(totalSize); - - UInt64 currentPackTotal = 0; - UInt64 currentUnpTotal = 0; - UInt64 currentPackSize = 0; - UInt64 currentUnpSize = 0; - - const UInt32 kZeroBufSize = (1 << 14); - CByteBuffer zeroBuf(kZeroBufSize); - memset(zeroBuf, 0, kZeroBufSize); - - NCompress::CCopyCoder *copyCoderSpec = new NCompress::CCopyCoder(); - CMyComPtr copyCoder = copyCoderSpec; + RINOK(extractCallback->SetTotal(totalSize)) - NCompress::NZlib::CDecoder *zlibCoderSpec = new NCompress::NZlib::CDecoder(); - CMyComPtr zlibCoder = zlibCoderSpec; - - NCompress::NBZip2::CDecoder *bzip2CoderSpec = new NCompress::NBZip2::CDecoder(); - CMyComPtr bzip2Coder = bzip2CoderSpec; - - NCompress::NDeflate::NDecoder::CCOMCoder *deflateCoderSpec = new NCompress::NDeflate::NDecoder::CCOMCoder(); - CMyComPtr deflateCoder = deflateCoderSpec; - - CLocalProgress *lps = new CLocalProgress; - CMyComPtr progress = lps; + CMyComPtr2_Create lps; lps->Init(extractCallback, false); - - CLimitedSequentialInStream *inStreamSpec = new CLimitedSequentialInStream; - CMyComPtr inStream(inStreamSpec); - inStreamSpec->SetStream(_inStream); - + CInStreamWithHash inHashStream; + CMyComPtr2_Create outStreamSha1; + CMyComPtr2_Create outStreamSha256; + CMyComPtr2_Create outStreamLim; + CMyComPtr2_Create copyCoder; + CMyComPtr2_Create zlibCoder; + CMyComPtr2_Create bzip2Coder; + bzip2Coder->FinishMode = true; - CLimitedSequentialOutStream *outStreamLimSpec = new CLimitedSequentialOutStream; - CMyComPtr outStream(outStreamLimSpec); + UInt64 cur_PackSize, cur_UnpSize; - COutStreamWithSha1 *outStreamSha1Spec = new COutStreamWithSha1; + for (i = 0;; i++, + lps->InSize += cur_PackSize, + lps->OutSize += cur_UnpSize) { - CMyComPtr outStreamSha1(outStreamSha1Spec); - outStreamLimSpec->SetStream(outStreamSha1); - } - - for (i = 0; i < numItems; i++, currentPackTotal += currentPackSize, currentUnpTotal += currentUnpSize) - { - lps->InSize = currentPackTotal; - lps->OutSize = currentUnpTotal; - currentPackSize = 0; - currentUnpSize = 0; + cur_PackSize = 0; + cur_UnpSize = 0; RINOK(lps->SetCur()) + if (i >= numItems) + break; + CMyComPtr realOutStream; const Int32 askMode = testMode ? NExtract::NAskMode::kTest : @@ -606,6 +1108,7 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, if (item.IsDir) { RINOK(extractCallback->PrepareOperation(askMode)) + realOutStream.Release(); RINOK(extractCallback->SetOperationResult(NExtract::NOperationResult::kOK)) continue; } @@ -615,39 +1118,63 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, continue; RINOK(extractCallback->PrepareOperation(askMode)) - outStreamSha1Spec->SetStream(realOutStream); - realOutStream.Release(); - Int32 opRes = NExtract::NOperationResult::kOK; - #ifdef XAR_SHOW_RAW - if (index == _files.Size()) + +#ifdef XAR_SHOW_RAW + if (index >= _files.Size()) { - outStreamSha1Spec->Init(false); - outStreamLimSpec->Init(_xmlLen); - RINOK(WriteStream(outStream, _xml, _xmlLen)) - currentPackSize = currentUnpSize = _xmlLen; + cur_PackSize = cur_UnpSize = _xmlLen; + if (realOutStream) + RINOK(WriteStream(realOutStream, _xmlBuf, _xmlLen)) + realOutStream.Release(); } else - #endif +#endif { const CFile &item = _files[index]; - if (item.HasData) + if (!item.HasData) + realOutStream.Release(); + else { - currentPackSize = item.PackSize; - currentUnpSize = item.Size; + cur_PackSize = item.PackSize; + cur_UnpSize = item.Size; RINOK(InStream_SeekSet(_inStream, _dataStartPos + item.Offset)) - inStreamSpec->Init(item.PackSize); - outStreamSha1Spec->Init(item.Sha1IsDefined); - outStreamLimSpec->Init(item.Size); + + inHashStream.SetStreamAndInit(_inStream, item.archived_checksum.AlgoNumber); + inHashStream.inStreamLim->Init(item.PackSize); + + const int checksum_method = item.extracted_checksum.AlgoNumber; + if (checksum_method == XAR_CKSUM_SHA1) + { + outStreamLim->SetStream(outStreamSha1); + outStreamSha1->SetStream(realOutStream); + outStreamSha1->Init(); + } + else if (checksum_method == XAR_CKSUM_SHA256) + { + outStreamLim->SetStream(outStreamSha256); + outStreamSha256->SetStream(realOutStream); + outStreamSha256->Init(); + } + else + outStreamLim->SetStream(realOutStream); + + realOutStream.Release(); + + // outStreamSha1->Init(item.Sha1IsDefined); + + outStreamLim->Init(item.Size); HRESULT res = S_OK; ICompressCoder *coder = NULL; if (item.IsCopyMethod()) + { if (item.PackSize == item.Size) coder = copyCoder; else opRes = NExtract::NOperationResult::kUnsupportedMethod; + } else if (item.Method == METHOD_NAME_ZLIB) coder = zlibCoder; else if (item.Method == "bzip2") @@ -656,11 +1183,11 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, opRes = NExtract::NOperationResult::kUnsupportedMethod; if (coder) - res = coder->Code(inStream, outStream, NULL, NULL, progress); + res = coder->Code(inHashStream.inStreamLim, outStreamLim, NULL, &item.Size, lps); if (res != S_OK) { - if (!outStreamLimSpec->IsFinishedOK()) + if (!outStreamLim->IsFinishedOK()) opRes = NExtract::NOperationResult::kDataError; else if (res != S_FALSE) return res; @@ -670,27 +1197,38 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, if (opRes == NExtract::NOperationResult::kOK) { - if (outStreamLimSpec->IsFinishedOK() && - outStreamSha1Spec->GetSize() == item.Size) + if (outStreamLim->IsFinishedOK()) { - if (!outStreamLimSpec->IsFinishedOK()) + if (checksum_method == XAR_CKSUM_SHA1) { - opRes = NExtract::NOperationResult::kDataError; + Byte digest[SHA1_DIGEST_SIZE]; + outStreamSha1->Final(digest); + if (memcmp(digest, item.extracted_checksum.Data, SHA1_DIGEST_SIZE) != 0) + opRes = NExtract::NOperationResult::kCRCError; } - else if (item.Sha1IsDefined) + else if (checksum_method == XAR_CKSUM_SHA256) { - Byte digest[SHA1_DIGEST_SIZE]; - outStreamSha1Spec->Final(digest); - if (memcmp(digest, item.Sha1, SHA1_DIGEST_SIZE) != 0) + Byte digest[SHA256_DIGEST_SIZE]; + outStreamSha256->Final(digest); + if (memcmp(digest, item.extracted_checksum.Data, SHA256_DIGEST_SIZE) != 0) opRes = NExtract::NOperationResult::kCRCError; } + if (opRes == NExtract::NOperationResult::kOK) + if (!inHashStream.CheckHash( + item.archived_checksum.AlgoNumber, + item.archived_checksum.Data)) + opRes = NExtract::NOperationResult::kCRCError; } else opRes = NExtract::NOperationResult::kDataError; } + if (checksum_method == XAR_CKSUM_SHA1) + outStreamSha1->ReleaseStream(); + else if (checksum_method == XAR_CKSUM_SHA256) + outStreamSha256->ReleaseStream(); } + outStreamLim->ReleaseStream(); } - outStreamSha1Spec->ReleaseStream(); RINOK(extractCallback->SetOperationResult(opRes)) } return S_OK; @@ -701,14 +1239,14 @@ Z7_COM7F_IMF(CHandler::GetStream(UInt32 index, ISequentialInStream **stream)) { *stream = NULL; COM_TRY_BEGIN - #ifdef XAR_SHOW_RAW - if (index == _files.Size()) +#ifdef XAR_SHOW_RAW + if (index >= _files.Size()) { - Create_BufInStream_WithNewBuffer(_xml, _xmlLen, stream); + Create_BufInStream_WithNewBuffer(_xmlBuf, _xmlLen, stream); return S_OK; } else - #endif +#endif { const CFile &item = _files[index]; if (item.HasData && item.IsCopyMethod() && item.PackSize == item.Size) @@ -718,7 +1256,12 @@ Z7_COM7F_IMF(CHandler::GetStream(UInt32 index, ISequentialInStream **stream)) COM_TRY_END } -static const Byte k_Signature[] = { 'x', 'a', 'r', '!', 0, 0x1C }; +// 0x1c == 28 is expected header size value for most archives. +// but we want to support another (rare case) headers sizes. +// so we must reduce signature to 4 or 5 bytes. +static const Byte k_Signature[] = +// { 'x', 'a', 'r', '!', 0, 0x1C, 0 }; + { 'x', 'a', 'r', '!', 0 }; REGISTER_ARC_I( "Xar", "xar pkg xip", NULL, 0xE1, diff --git a/CPP/7zip/Archive/XzHandler.cpp b/CPP/7zip/Archive/XzHandler.cpp index 976817c..7ced4e1 100644 --- a/CPP/7zip/Archive/XzHandler.cpp +++ b/CPP/7zip/Archive/XzHandler.cpp @@ -76,11 +76,15 @@ struct CBlockInfo Z7_IFACE_COM7_IMP(IOutArchive) #endif - CXzStatInfo _stat; // it's stat from backward parsing - CXzStatInfo _stat2; // it's data from forward parsing, if the decoder was called - SRes _stat2_decode_SRes; bool _stat_defined; bool _stat2_defined; + bool _isArc; + bool _needSeekToStart; + bool _firstBlockWasRead; + SRes _stat2_decode_SRes; + + CXzStatInfo _stat; // it's stat from backward parsing + CXzStatInfo _stat2; // it's data from forward parsing, if the decoder was called const CXzStatInfo *GetStat() const { @@ -89,10 +93,6 @@ struct CBlockInfo return NULL; } - bool _isArc; - bool _needSeekToStart; - bool _firstBlockWasRead; - AString _methodsString; @@ -134,7 +134,7 @@ struct CBlockInfo #endif decoder._memUsage = _memUsage_Decompress; - HRESULT hres = decoder.Decode(seqInStream, outStream, + const HRESULT hres = decoder.Decode(seqInStream, outStream, NULL, // *outSizeLimit true, // finishStream progress); @@ -151,6 +151,11 @@ struct CBlockInfo } } + if (hres == S_OK && progress) + { + // RINOK( + progress->SetRatioInfo(&decoder.Stat.InSize, &decoder.Stat.OutSize); + } return hres; } @@ -207,17 +212,6 @@ static const Byte kArcProps[] = IMP_IInArchive_Props IMP_IInArchive_ArcProps -static inline char GetHex(unsigned value) -{ - return (char)((value < 10) ? ('0' + value) : ('A' + (value - 10))); -} - -static inline void AddHexToString(AString &s, Byte value) -{ - s += GetHex(value >> 4); - s += GetHex(value & 0xF); -} - static void Lzma2PropToString(AString &s, unsigned prop) { char c = 0; @@ -236,7 +230,7 @@ static void Lzma2PropToString(AString &s, unsigned prop) } s.Add_UInt32(size); if (c != 0) - s += c; + s.Add_Char(c); } struct CMethodNamePair @@ -256,6 +250,7 @@ static const CMethodNamePair g_NamePairs[] = { XZ_ID_ARMT, "ARMT" }, { XZ_ID_SPARC, "SPARC" }, { XZ_ID_ARM64, "ARM64" }, + { XZ_ID_RISCV, "RISCV" }, { XZ_ID_LZMA2, "LZMA2" } }; @@ -279,7 +274,7 @@ static void AddMethodString(AString &s, const CXzFilter &f) if (f.propsSize > 0) { - s += ':'; + s.Add_Colon(); if (f.id == XZ_ID_LZMA2 && f.propsSize == 1) Lzma2PropToString(s, f.props[0]); else if (f.id == XZ_ID_Delta && f.propsSize == 1) @@ -288,10 +283,14 @@ static void AddMethodString(AString &s, const CXzFilter &f) s.Add_UInt32((UInt32)f.props[0] + 16 + 2); else { - s += '['; + s.Add_Char('['); for (UInt32 bi = 0; bi < f.propsSize; bi++) - AddHexToString(s, f.props[bi]); - s += ']'; + { + const unsigned v = f.props[bi]; + s.Add_Char(GET_HEX_CHAR_UPPER(v >> 4)); + s.Add_Char(GET_HEX_CHAR_UPPER(v & 15)); + } + s.Add_Char(']'); } } } @@ -387,6 +386,7 @@ Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) // if (_blocks) prop = (UInt32)0; break; } + default: break; } prop.Detach(value); return S_OK; @@ -409,6 +409,7 @@ Z7_COM7F_IMF(CHandler::GetProperty(UInt32, PROPID propID, PROPVARIANT *value)) case kpidSize: if (stat && stat->UnpackSize_Defined) prop = stat->OutSize; break; case kpidPackSize: if (stat) prop = stat->InSize; break; case kpidMethod: if (!_methodsString.IsEmpty()) prop = _methodsString; break; + default: break; } prop.Detach(value); return S_OK; @@ -488,6 +489,7 @@ static HRESULT SRes_to_Open_HRESULT(SRes res) case SZ_ERROR_NO_ARCHIVE: return S_FALSE; */ + default: break; } return S_FALSE; } @@ -737,11 +739,9 @@ CXzUnpackerCPP2::~CXzUnpackerCPP2() } -Z7_CLASS_IMP_COM_1( +Z7_CLASS_IMP_IInStream( CInStream - , IInStream ) - Z7_IFACE_COM7_IMP(ISequentialInStream) UInt64 _virtPos; public: @@ -760,9 +760,7 @@ Z7_CLASS_IMP_COM_1( // _startPos = startPos; } - CHandler *_handlerSpec; - CMyComPtr _handler; - + CMyComPtr2 _handlerSpec; // ~CInStream(); }; @@ -922,7 +920,7 @@ Z7_COM7F_IMF(CInStream::Read(void *data, UInt32 size, UInt32 *processedSize)) const size_t rem = _cacheSize - offset; if (size > rem) size = (UInt32)rem; - memcpy(data, _cache + offset, size); + memcpy(data, _cache.ConstData() + offset, size); _virtPos += size; if (processedSize) *processedSize = size; @@ -977,15 +975,15 @@ Z7_COM7F_IMF(CHandler::GetStream(UInt32 index, ISequentialInStream **stream)) return S_FALSE; } - CInStream *spec = new CInStream; - CMyComPtr specStream = spec; + CMyComPtr2 spec; + spec.Create_if_Empty(); spec->_cache.Alloc((size_t)_maxBlocksSize); - spec->_handlerSpec = this; - spec->_handler = (IInArchive *)this; + spec->_handlerSpec.SetFromCls(this); + // spec->_handler = (IInArchive *)this; spec->Size = _stat.OutSize; spec->InitAndSeek(); - *stream = specStream.Detach(); + *stream = spec.Detach(); return S_OK; COM_TRY_END @@ -1032,10 +1030,12 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, const CXzStatInfo *stat = GetStat(); if (stat) - extractCallback->SetTotal(stat->InSize); + RINOK(extractCallback->SetTotal(stat->InSize)) UInt64 currentTotalPacked = 0; RINOK(extractCallback->SetCompleted(¤tTotalPacked)) + Int32 opRes; + { CMyComPtr realOutStream; const Int32 askMode = testMode ? NExtract::NAskMode::kTest : @@ -1046,10 +1046,9 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, if (!testMode && !realOutStream) return S_OK; - extractCallback->PrepareOperation(askMode); + RINOK(extractCallback->PrepareOperation(askMode)) - CLocalProgress *lps = new CLocalProgress; - CMyComPtr lpsRef = lps; + CMyComPtr2_Create lps; lps->Init(extractCallback, true); if (_needSeekToStart) @@ -1064,17 +1063,18 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, NCompress::NXz::CDecoder decoder; - HRESULT hres = Decode(decoder, _seqStream, realOutStream, lpsRef); + const HRESULT hres = Decode(decoder, _seqStream, realOutStream, lps); if (!decoder.MainDecodeSRes_wasUsed) return hres == S_OK ? E_FAIL : hres; - Int32 opRes = Get_Extract_OperationResult(decoder); + opRes = Get_Extract_OperationResult(decoder); if (opRes == NExtract::NOperationResult::kOK && hres != S_OK) opRes = NExtract::NOperationResult::kDataError; - realOutStream.Release(); + // realOutStream.Release(); + } return extractCallback->SetOperationResult(opRes); COM_TRY_END } @@ -1143,10 +1143,9 @@ Z7_COM7F_IMF(CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt dataSize = prop.uhVal.QuadPart; } - NCompress::NXz::CEncoder *encoderSpec = new NCompress::NXz::CEncoder; - CMyComPtr encoder = encoderSpec; + CMyComPtr2_Create encoder; - CXzProps &xzProps = encoderSpec->xzProps; + CXzProps &xzProps = encoder->xzProps; CLzma2EncProps &lzma2Props = xzProps.lzma2Props; lzma2Props.lzmaProps.level = GetLevel(); @@ -1155,7 +1154,7 @@ Z7_COM7F_IMF(CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt /* { NCOM::CPropVariant prop = (UInt64)dataSize; - RINOK(encoderSpec->SetCoderProp(NCoderPropID::kReduceSize, prop)) + RINOK(encoder->SetCoderProp(NCoderPropID::kReduceSize, prop)) } */ @@ -1231,7 +1230,7 @@ Z7_COM7F_IMF(CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt xzProps.lzma2Props.blockSize = LZMA2_ENC_PROPS_BLOCK_SIZE_SOLID; } - RINOK(encoderSpec->SetCheckSize(_crcSize)) + RINOK(encoder->SetCheckSize(_crcSize)) { CXzFilterProps &filter = xzProps.filterProps; @@ -1266,7 +1265,7 @@ Z7_COM7F_IMF(CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt FOR_VECTOR (j, m.Props) { const CProp &prop = m.Props[j]; - RINOK(encoderSpec->SetCoderProp(prop.Id, prop.Value)) + RINOK(encoder->SetCoderProp(prop.Id, prop.Value)) } } @@ -1286,10 +1285,9 @@ Z7_COM7F_IMF(CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt } } RINOK(updateCallback->SetTotal(dataSize)) - CLocalProgress *lps = new CLocalProgress; - CMyComPtr progress = lps; + CMyComPtr2_Create lps; lps->Init(updateCallback, true); - RINOK(encoder->Code(fileInStream, outStream, NULL, NULL, progress)) + RINOK(encoder.Interface()->Code(fileInStream, outStream, NULL, NULL, lps)) } return updateCallback->SetOperationResult(NArchive::NUpdate::NOperationResult::kOK); @@ -1316,11 +1314,10 @@ Z7_COM7F_IMF(CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt RINOK(InStream_SeekToBegin(_stream)) } - CLocalProgress *lps = new CLocalProgress; - CMyComPtr progress = lps; + CMyComPtr2_Create lps; lps->Init(updateCallback, true); - return NCompress::CopyStream(_stream, outStream, progress); + return NCompress::CopyStream(_stream, outStream, lps); COM_TRY_END } diff --git a/CPP/7zip/Archive/ZHandler.cpp b/CPP/7zip/Archive/ZHandler.cpp index 18e712a..1caef34 100644 --- a/CPP/7zip/Archive/ZHandler.cpp +++ b/CPP/7zip/Archive/ZHandler.cpp @@ -89,7 +89,7 @@ API_FUNC_static_IsArc IsArc_Z(const Byte *p, size_t size) if (size < 3) return k_IsArc_Res_NEED_MORE; if (size > NCompress::NZ::kRecommendedCheckSize) - size = NCompress::NZ::kRecommendedCheckSize; + size = NCompress::NZ::kRecommendedCheckSize; if (!NCompress::NZ::CheckStream(p, size)) return k_IsArc_Res_NO; return k_IsArc_Res_YES; @@ -166,13 +166,12 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, return S_OK; if (numItems != (UInt32)(Int32)-1 && (numItems != 1 || indices[0] != 0)) return E_INVALIDARG; - - extractCallback->SetTotal(_packSize); - + RINOK(extractCallback->SetTotal(_packSize)) UInt64 currentTotalPacked = 0; - RINOK(extractCallback->SetCompleted(¤tTotalPacked)) + int opRes; + { CMyComPtr realOutStream; const Int32 askMode = testMode ? NExtract::NAskMode::kTest : @@ -183,37 +182,33 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, if (!testMode && !realOutStream) return S_OK; - extractCallback->PrepareOperation(askMode); + RINOK(extractCallback->PrepareOperation(askMode)) - CDummyOutStream *outStreamSpec = new CDummyOutStream; - CMyComPtr outStream(outStreamSpec); - outStreamSpec->SetStream(realOutStream); - outStreamSpec->Init(); - realOutStream.Release(); + CMyComPtr2_Create outStream; + outStream->SetStream(realOutStream); + outStream->Init(); + // realOutStream.Release(); - CLocalProgress *lps = new CLocalProgress; - CMyComPtr progress = lps; + CMyComPtr2_Create lps; lps->Init(extractCallback, true); RINOK(InStream_SeekToBegin(_stream)) - NCompress::NZ::CDecoder *decoderSpec = new NCompress::NZ::CDecoder; - CMyComPtr decoder = decoderSpec; - - int opRes; + NCompress::NZ::CDecoder decoder; { - HRESULT result = decoder->Code(_stream, outStream, NULL, NULL, progress); - if (result == S_FALSE) + const HRESULT hres = decoder.Code(_stream, outStream, lps); + if (hres == S_FALSE) opRes = NExtract::NOperationResult::kDataError; else { - RINOK(result) + RINOK(hres) opRes = NExtract::NOperationResult::kOK; } } // _unpackSize = outStreamSpec->GetSize(); // _unpackSize_Defined = true; - outStream.Release(); + // outStream.Release(); + } return extractCallback->SetOperationResult(opRes); COM_TRY_END } diff --git a/CPP/7zip/Archive/Zip/ZipAddCommon.cpp b/CPP/7zip/Archive/Zip/ZipAddCommon.cpp index 8af84b4..43d68c7 100644 --- a/CPP/7zip/Archive/Zip/ZipAddCommon.cpp +++ b/CPP/7zip/Archive/Zip/ZipAddCommon.cpp @@ -40,24 +40,18 @@ Z7_CLASS_IMP_NOQIB_3( , ICompressSetCoderPropertiesOpt ) public: - NCompress::NLzma::CEncoder *EncoderSpec; - CMyComPtr Encoder; + CMyComPtr2 Encoder; Byte Header[kLzmaHeaderSize]; }; Z7_COM7F_IMF(CLzmaEncoder::SetCoderProperties(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps)) { - if (!Encoder) - { - EncoderSpec = new NCompress::NLzma::CEncoder; - Encoder = EncoderSpec; - } - CBufPtrSeqOutStream *outStreamSpec = new CBufPtrSeqOutStream; - CMyComPtr outStream(outStreamSpec); - outStreamSpec->Init(Header + 4, kLzmaPropsSize); - RINOK(EncoderSpec->SetCoderProperties(propIDs, props, numProps)) - RINOK(EncoderSpec->WriteCoderProperties(outStream)) - if (outStreamSpec->GetPos() != kLzmaPropsSize) + Encoder.Create_if_Empty(); + CMyComPtr2_Create outStream; + outStream->Init(Header + 4, kLzmaPropsSize); + RINOK(Encoder->SetCoderProperties(propIDs, props, numProps)) + RINOK(Encoder->WriteCoderProperties(outStream)) + if (outStream->GetPos() != kLzmaPropsSize) return E_FAIL; Header[0] = MY_VER_MAJOR; Header[1] = MY_VER_MINOR; @@ -68,21 +62,19 @@ Z7_COM7F_IMF(CLzmaEncoder::SetCoderProperties(const PROPID *propIDs, const PROPV Z7_COM7F_IMF(CLzmaEncoder::SetCoderPropertiesOpt(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps)) { - return EncoderSpec->SetCoderPropertiesOpt(propIDs, props, numProps); + return Encoder->SetCoderPropertiesOpt(propIDs, props, numProps); } Z7_COM7F_IMF(CLzmaEncoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress)) { RINOK(WriteStream(outStream, Header, kLzmaHeaderSize)) - return Encoder->Code(inStream, outStream, inSize, outSize, progress); + return Encoder.Interface()->Code(inStream, outStream, inSize, outSize, progress); } CAddCommon::CAddCommon(): - _copyCoderSpec(NULL), _isLzmaEos(false), - _cryptoStreamSpec(NULL), _buf(NULL) {} @@ -181,9 +173,10 @@ HRESULT CAddCommon::Set_Pre_CompressionResult(bool inSeqMode, bool outSeqMode, U opRes.LzmaEos = oneMethodMain->Get_Lzma_Eos(); break; } + default: break; } if (opRes.ExtractVersion < ver) - opRes.ExtractVersion = ver; + opRes.ExtractVersion = ver; return S_OK; } @@ -205,8 +198,7 @@ HRESULT CAddCommon::Compress( return E_INVALIDARG; } - CSequentialInStreamWithCRC *inSecCrcStreamSpec = new CSequentialInStreamWithCRC; - CMyComPtr inCrcStream = inSecCrcStreamSpec; + CMyComPtr2_Create inCrcStream; CMyComPtr inStream2; if (!inSeqMode) @@ -220,9 +212,9 @@ HRESULT CAddCommon::Compress( } } - inSecCrcStreamSpec->SetStream(inStream); - inSecCrcStreamSpec->SetFullSize(expectedDataSize_IsConfirmed ? expectedDataSize : (UInt64)(Int64)-1); - // inSecCrcStreamSpec->Init(); + inCrcStream->SetStream(inStream); + inCrcStream->SetFullSize(expectedDataSize_IsConfirmed ? expectedDataSize : (UInt64)(Int64)-1); + // inCrcStream->Init(); unsigned numTestMethods = _options.MethodSequence.Size(); // numTestMethods != 0 @@ -250,7 +242,7 @@ HRESULT CAddCommon::Compress( for (unsigned i = 0; i < numTestMethods; i++) { - inSecCrcStreamSpec->Init(); + inCrcStream->Init(); if (i != 0) { @@ -278,18 +270,15 @@ HRESULT CAddCommon::Compress( { opRes.ExtractVersion = NCompressionMethod::kExtractVersion_ZipCrypto; - if (!_cryptoStream) - { - _cryptoStreamSpec = new CFilterCoder(true); - _cryptoStream = _cryptoStreamSpec; - } + if (!_cryptoStream.IsDefined()) + _cryptoStream.SetFromCls(new CFilterCoder(true)); if (_options.IsAesMode) { opRes.ExtractVersion = NCompressionMethod::kExtractVersion_Aes; - if (!_cryptoStreamSpec->Filter) + if (!_cryptoStream->Filter) { - _cryptoStreamSpec->Filter = _filterAesSpec = new NCrypto::NWzAes::CEncoder; + _cryptoStream->Filter = _filterAesSpec = new NCrypto::NWzAes::CEncoder; _filterAesSpec->SetKeyMode(_options.AesKeyMode); RINOK(_filterAesSpec->CryptoSetPassword((const Byte *)(const char *)_options.Password, _options.Password.Len())) } @@ -297,9 +286,9 @@ HRESULT CAddCommon::Compress( } else { - if (!_cryptoStreamSpec->Filter) + if (!_cryptoStream->Filter) { - _cryptoStreamSpec->Filter = _filterSpec = new NCrypto::NZip::CEncoder; + _cryptoStream->Filter = _filterSpec = new NCrypto::NZip::CEncoder; _filterSpec->CryptoSetPassword((const Byte *)(const char *)_options.Password, _options.Password.Len()); } @@ -317,7 +306,7 @@ HRESULT CAddCommon::Compress( RINOK(CalcStreamCRC(inStream, crc)) crc_IsCalculated = true; RINOK(InStream_SeekToBegin(inStream2)) - inSecCrcStreamSpec->Init(); + inCrcStream->Init(); } check = (crc >> 16); } @@ -328,13 +317,13 @@ HRESULT CAddCommon::Compress( if (method == NCompressionMethod::kStore) { needCode = false; - RINOK(_cryptoStreamSpec->Code(inCrcStream, outStream, NULL, NULL, progress)) + RINOK(_cryptoStream->Code(inCrcStream, outStream, NULL, NULL, progress)) } else { - RINOK(_cryptoStreamSpec->SetOutStream(outStream)) - RINOK(_cryptoStreamSpec->InitEncoder()) - outStreamReleaser.FilterCoder = _cryptoStreamSpec; + RINOK(_cryptoStream->SetOutStream(outStream)) + RINOK(_cryptoStream->InitEncoder()) + outStreamReleaser.FilterCoder = _cryptoStream.ClsPtr(); } } @@ -344,17 +333,13 @@ HRESULT CAddCommon::Compress( { case NCompressionMethod::kStore: { - if (!_copyCoderSpec) - { - _copyCoderSpec = new NCompress::CCopyCoder; - _copyCoder = _copyCoderSpec; - } + _copyCoder.Create_if_Empty(); CMyComPtr outStreamNew; if (_options.Password_Defined) outStreamNew = _cryptoStream; else outStreamNew = outStream; - RINOK(_copyCoder->Code(inCrcStream, outStreamNew, NULL, NULL, progress)) + RINOK(_copyCoder.Interface()->Code(inCrcStream, outStreamNew, NULL, NULL, progress)) break; } @@ -426,7 +411,7 @@ HRESULT CAddCommon::Compress( } } if (method == NCompressionMethod::kLZMA) - _isLzmaEos = _lzmaEncoder->EncoderSpec->IsWriteEndMark(); + _isLzmaEos = _lzmaEncoder->Encoder->IsWriteEndMark(); } if (method == NCompressionMethod::kLZMA) @@ -460,7 +445,7 @@ HRESULT CAddCommon::Compress( if (_options.Password_Defined) { - RINOK(_cryptoStreamSpec->OutStreamFinish()) + RINOK(_cryptoStream->OutStreamFinish()) } } @@ -475,12 +460,12 @@ HRESULT CAddCommon::Compress( RINOK(outStream->Seek(0, STREAM_SEEK_CUR, &opRes.PackSize)) { - opRes.CRC = inSecCrcStreamSpec->GetCRC(); - opRes.UnpackSize = inSecCrcStreamSpec->GetSize(); + opRes.CRC = inCrcStream->GetCRC(); + opRes.UnpackSize = inCrcStream->GetSize(); opRes.Method = method; } - if (!inSecCrcStreamSpec->WasFinished()) + if (!inCrcStream->WasFinished()) return E_FAIL; if (_options.Password_Defined) diff --git a/CPP/7zip/Archive/Zip/ZipAddCommon.h b/CPP/7zip/Archive/Zip/ZipAddCommon.h index 051915c..e72ec7b 100644 --- a/CPP/7zip/Archive/Zip/ZipAddCommon.h +++ b/CPP/7zip/Archive/Zip/ZipAddCommon.h @@ -39,15 +39,13 @@ struct CCompressingResult class CAddCommon MY_UNCOPYABLE { CCompressionMethodMode _options; - NCompress::CCopyCoder *_copyCoderSpec; - CMyComPtr _copyCoder; + CMyComPtr2 _copyCoder; CMyComPtr _compressEncoder; Byte _compressExtractVersion; bool _isLzmaEos; - CFilterCoder *_cryptoStreamSpec; - CMyComPtr _cryptoStream; + CMyComPtr2 _cryptoStream; NCrypto::NZip::CEncoder *_filterSpec; NCrypto::NWzAes::CEncoder *_filterAesSpec; diff --git a/CPP/7zip/Archive/Zip/ZipHandler.cpp b/CPP/7zip/Archive/Zip/ZipHandler.cpp index 5f022cc..c6b6e39 100644 --- a/CPP/7zip/Archive/Zip/ZipHandler.cpp +++ b/CPP/7zip/Archive/Zip/ZipHandler.cpp @@ -18,22 +18,15 @@ #include "../../Common/StreamUtils.h" #include "../../Compress/CopyCoder.h" - -#ifdef Z7_EXTERNAL_CODECS -#ifndef SUPPORT_LZFSE -#define SUPPORT_LZFSE -#endif -#endif - -#ifdef SUPPORT_LZFSE +#ifndef Z7_ZIP_LZFSE_DISABLE #include "../../Compress/LzfseDecoder.h" #endif - #include "../../Compress/LzmaDecoder.h" #include "../../Compress/ImplodeDecoder.h" #include "../../Compress/PpmdZip.h" #include "../../Compress/ShrinkDecoder.h" #include "../../Compress/XzDecoder.h" +#include "../../Compress/ZstdDecoder.h" #include "../../Crypto/WzAes.h" #include "../../Crypto/ZipCrypto.h" @@ -341,6 +334,7 @@ Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) } // case kpidIsAltStream: prop = true; break; + default: break; } return prop.Detach(value); COM_TRY_END @@ -546,7 +540,7 @@ Z7_COM7F_IMF(CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val else { m += kMethod_StrongCrypto; - m += ':'; + m.Add_Colon(); m.Add_UInt32(f.AlgId); } if (f.CertificateIsUsed()) @@ -567,7 +561,7 @@ Z7_COM7F_IMF(CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val s = kMethodNames1[id]; else { - int id2 = (int)id - (int)kMethodNames2Start; + const int id2 = (int)id - (int)kMethodNames2Start; if (id2 >= 0 && (unsigned)id2 < kNumMethodNames2) s = kMethodNames2[id2]; } @@ -588,7 +582,7 @@ Z7_COM7F_IMF(CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val } else if (id == NFileHeader::NCompressionMethod::kDeflate) { - m += ':'; + m.Add_Colon(); m += kDeflateLevels[level]; level = 0; } @@ -684,6 +678,7 @@ Z7_COM7F_IMF(CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val } break; */ + default: break; } return prop.Detach(value); @@ -808,18 +803,9 @@ Z7_CLASS_IMP_NOQIB_3( , ICompressGetInStreamProcessedSize ) public: - NCompress::NLzma::CDecoder *DecoderSpec; - CMyComPtr Decoder; - - CLzmaDecoder(); + CMyComPtr2_Create Decoder; }; -CLzmaDecoder::CLzmaDecoder() -{ - DecoderSpec = new NCompress::NLzma::CDecoder; - Decoder = DecoderSpec; -} - static const unsigned kZipLzmaPropsSize = 4 + LZMA_PROPS_SIZE; Z7_COM7F_IMF(CLzmaDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, @@ -829,7 +815,7 @@ Z7_COM7F_IMF(CLzmaDecoder::Code(ISequentialInStream *inStream, ISequentialOutStr RINOK(ReadStream_FALSE(inStream, buf, kZipLzmaPropsSize)) if (buf[2] != LZMA_PROPS_SIZE || buf[3] != 0) return E_NOTIMPL; - RINOK(DecoderSpec->SetDecoderProperties2(buf + 4, LZMA_PROPS_SIZE)) + RINOK(Decoder->SetDecoderProperties2(buf + 4, LZMA_PROPS_SIZE)) UInt64 inSize2 = 0; if (inSize) { @@ -838,18 +824,18 @@ Z7_COM7F_IMF(CLzmaDecoder::Code(ISequentialInStream *inStream, ISequentialOutStr return S_FALSE; inSize2 -= kZipLzmaPropsSize; } - return Decoder->Code(inStream, outStream, inSize ? &inSize2 : NULL, outSize, progress); + return Decoder.Interface()->Code(inStream, outStream, inSize ? &inSize2 : NULL, outSize, progress); } Z7_COM7F_IMF(CLzmaDecoder::SetFinishMode(UInt32 finishMode)) { - DecoderSpec->FinishStream = (finishMode != 0); + Decoder->FinishStream = (finishMode != 0); return S_OK; } Z7_COM7F_IMF(CLzmaDecoder::GetInStreamProcessedSize(UInt64 *value)) { - *value = DecoderSpec->GetInputProcessedSize() + kZipLzmaPropsSize; + *value = Decoder->GetInputProcessedSize() + kZipLzmaPropsSize; return S_OK; } @@ -869,26 +855,17 @@ struct CMethodItem class CZipDecoder { - NCrypto::NZip::CDecoder *_zipCryptoDecoderSpec; - NCrypto::NZipStrong::CDecoder *_pkAesDecoderSpec; - NCrypto::NWzAes::CDecoder *_wzAesDecoderSpec; - - CMyComPtr _zipCryptoDecoder; - CMyComPtr _pkAesDecoder; - CMyComPtr _wzAesDecoder; + CMyComPtr2 _zipCryptoDecoder; + CMyComPtr2 _pkAesDecoder; + CMyComPtr2 _wzAesDecoder; - CFilterCoder *filterStreamSpec; - CMyComPtr filterStream; + CMyComPtr2 filterStream; CMyComPtr getTextPassword; CObjectVector methodItems; CLzmaDecoder *lzmaDecoderSpec; public: CZipDecoder(): - _zipCryptoDecoderSpec(NULL), - _pkAesDecoderSpec(NULL), - _wzAesDecoderSpec(NULL), - filterStreamSpec(NULL), lzmaDecoderSpec(NULL) {} @@ -1051,15 +1028,12 @@ HRESULT CZipDecoder::Decode( } } - COutStreamWithCRC *outStreamSpec = new COutStreamWithCRC; - CMyComPtr outStream = outStreamSpec; - outStreamSpec->SetStream(realOutStream); - outStreamSpec->Init(needCRC); + CMyComPtr2_Create outStream; + outStream->SetStream(realOutStream); + outStream->Init(needCRC); CMyComPtr packStream; - - CLimitedSequentialInStream *limitedStreamSpec = new CLimitedSequentialInStream; - CMyComPtr inStream(limitedStreamSpec); + CMyComPtr2_Create inStream; { UInt64 packSize = item.PackSize; @@ -1075,8 +1049,8 @@ HRESULT CZipDecoder::Decode( res = NExtract::NOperationResult::kUnavailable; return S_OK; } - limitedStreamSpec->SetStream(packStream); - limitedStreamSpec->Init(packSize); + inStream->SetStream(packStream); + inStream->Init(packSize); } @@ -1089,13 +1063,9 @@ HRESULT CZipDecoder::Decode( if (wzAesMode) { id = aesField.Method; - if (!_wzAesDecoder) - { - _wzAesDecoderSpec = new NCrypto::NWzAes::CDecoder; - _wzAesDecoder = _wzAesDecoderSpec; - } + _wzAesDecoder.Create_if_Empty(); cryptoFilter = _wzAesDecoder; - if (!_wzAesDecoderSpec->SetKeyMode(aesField.Strength)) + if (!_wzAesDecoder->SetKeyMode(aesField.Strength)) { res = NExtract::NOperationResult::kUnsupportedMethod; return S_OK; @@ -1103,20 +1073,12 @@ HRESULT CZipDecoder::Decode( } else if (pkAesMode) { - if (!_pkAesDecoder) - { - _pkAesDecoderSpec = new NCrypto::NZipStrong::CDecoder; - _pkAesDecoder = _pkAesDecoderSpec; - } + _pkAesDecoder.Create_if_Empty(); cryptoFilter = _pkAesDecoder; } else { - if (!_zipCryptoDecoder) - { - _zipCryptoDecoderSpec = new NCrypto::NZip::CDecoder; - _zipCryptoDecoder = _zipCryptoDecoderSpec; - } + _zipCryptoDecoder.Create_if_Empty(); cryptoFilter = _zipCryptoDecoder; } @@ -1135,15 +1097,17 @@ HRESULT CZipDecoder::Decode( AString_Wipe charPassword; if (password) { - /* - // 22.00: do we need UTF-8 passwords here ? - if (item.IsUtf8()) // 22.00 +#if 0 && defined(_WIN32) + // do we need UTF-8 passwords here ? + if (item.GetHostOS() == NFileHeader::NHostOS::kUnix // 24.05 + // || item.IsUtf8() // 22.00 + ) { // throw 1; ConvertUnicodeToUTF8((LPCOLESTR)password, charPassword); } else - */ +#endif { UnicodeStringToMultiByte2(charPassword, (LPCOLESTR)password, CP_ACP); } @@ -1202,10 +1166,12 @@ HRESULT CZipDecoder::Decode( mi.Coder = new NCompress::NXz::CComDecoder; else if (id == NFileHeader::NCompressionMethod::kPPMd) mi.Coder = new NCompress::NPpmdZip::CDecoder(true); - #ifdef SUPPORT_LZFSE + else if (id == NFileHeader::NCompressionMethod::kZstdWz) + mi.Coder = new NCompress::NZstd::CDecoder(); +#ifndef Z7_ZIP_LZFSE_DISABLE else if (id == NFileHeader::NCompressionMethod::kWzAES) mi.Coder = new NCompress::NLzfse::CDecoder; - #endif +#endif else { CMethodId szMethodID; @@ -1288,21 +1254,18 @@ HRESULT CZipDecoder::Decode( HRESULT result = S_OK; if (item.IsEncrypted()) { - if (!filterStream) - { - filterStreamSpec = new CFilterCoder(false); - filterStream = filterStreamSpec; - } + if (!filterStream.IsDefined()) + filterStream.SetFromCls(new CFilterCoder(false)); - filterReleaser.FilterCoder = filterStreamSpec; - filterStreamSpec->Filter = cryptoFilter; + filterReleaser.FilterCoder = filterStream.ClsPtr(); + filterStream->Filter = cryptoFilter; if (wzAesMode) { - result = _wzAesDecoderSpec->ReadHeader(inStream); + result = _wzAesDecoder->ReadHeader(inStream); if (result == S_OK) { - if (!_wzAesDecoderSpec->Init_and_CheckPassword()) + if (!_wzAesDecoder->Init_and_CheckPassword()) { res = NExtract::NOperationResult::kWrongPassword; return S_OK; @@ -1312,11 +1275,11 @@ HRESULT CZipDecoder::Decode( else if (pkAesMode) { isFullStreamExpected = false; - result =_pkAesDecoderSpec->ReadHeader(inStream, item.Crc, item.Size); + result = _pkAesDecoder->ReadHeader(inStream, item.Crc, item.Size); if (result == S_OK) { bool passwOK; - result = _pkAesDecoderSpec->Init_and_CheckPassword(passwOK); + result = _pkAesDecoder->Init_and_CheckPassword(passwOK); if (result == S_OK && !passwOK) { res = NExtract::NOperationResult::kWrongPassword; @@ -1326,10 +1289,10 @@ HRESULT CZipDecoder::Decode( } else { - result = _zipCryptoDecoderSpec->ReadHeader(inStream); + result = _zipCryptoDecoder->ReadHeader(inStream); if (result == S_OK) { - _zipCryptoDecoderSpec->Init_BeforeDecode(); + _zipCryptoDecoder->Init_BeforeDecode(); /* Info-ZIP modification to ZipCrypto format: if bit 3 of the general purpose bit flag is set, @@ -1337,10 +1300,10 @@ HRESULT CZipDecoder::Decode( Info-ZIP code probably writes 2 bytes of File Time. We check only 1 byte. */ - // UInt32 v1 = GetUi16(_zipCryptoDecoderSpec->_header + NCrypto::NZip::kHeaderSize - 2); + // UInt32 v1 = GetUi16(_zipCryptoDecoder->_header + NCrypto::NZip::kHeaderSize - 2); // UInt32 v2 = (item.HasDescriptor() ? (item.Time & 0xFFFF) : (item.Crc >> 16)); - Byte v1 = _zipCryptoDecoderSpec->_header[NCrypto::NZip::kHeaderSize - 1]; + Byte v1 = _zipCryptoDecoder->_header[NCrypto::NZip::kHeaderSize - 1]; Byte v2 = (Byte)(item.HasDescriptor() ? (item.Time >> 8) : (item.Crc >> 24)); if (v1 != v2) @@ -1361,7 +1324,7 @@ HRESULT CZipDecoder::Decode( RINOK(setFinishMode->SetFinishMode(BoolToUInt(true))) } - const UInt64 coderPackSize = limitedStreamSpec->GetRem(); + const UInt64 coderPackSize = inStream->GetRem(); if (id == NFileHeader::NCompressionMethod::kStore && item.IsEncrypted()) { @@ -1377,7 +1340,7 @@ HRESULT CZipDecoder::Decode( { padStreamSpec = new COutStreamWithPadPKCS7; padStream = padStreamSpec; - padSize = _pkAesDecoderSpec->GetPadSize((UInt32)item.Size); + padSize = _pkAesDecoder->GetPadSize((UInt32)item.Size); padStreamSpec->SetStream(outStream); padStreamSpec->Init(item.Size, padSize); } @@ -1394,12 +1357,12 @@ HRESULT CZipDecoder::Decode( size = expectedSize; } - result = filterStreamSpec->Code(inStream, padStream ? - (ISequentialOutStream *)padStream : - (ISequentialOutStream *)outStream, + result = filterStream->Code(inStream, padStream ? + padStream.Interface() : + outStream.Interface(), NULL, &size, compressProgress); - if (outStreamSpec->GetSize() != item.Size) + if (outStream->GetSize() != item.Size) truncatedError = true; if (pkAesMode) @@ -1415,20 +1378,20 @@ HRESULT CZipDecoder::Decode( if (item.IsEncrypted()) { readFromFilter = true; - inStreamReleaser.FilterCoder = filterStreamSpec; - RINOK(filterStreamSpec->SetInStream(inStream)) + inStreamReleaser.FilterCoder = filterStream.ClsPtr(); + RINOK(filterStream->SetInStream(inStream)) /* IFilter::Init() does nothing in all zip crypto filters. So we can call any Initialize function in CFilterCoder. */ - RINOK(filterStreamSpec->Init_NoSubFilterInit()) - // RINOK(filterStreamSpec->SetOutStreamSize(NULL)); + RINOK(filterStream->Init_NoSubFilterInit()) + // RINOK(filterStream->SetOutStreamSize(NULL)); } try { result = coder->Code(readFromFilter ? - (ISequentialInStream *)filterStream : - (ISequentialInStream *)inStream, + filterStream.Interface() : + inStream.Interface(), outStream, isFullStreamExpected ? &coderPackSize : NULL, // NULL, @@ -1448,7 +1411,7 @@ HRESULT CZipDecoder::Decode( { if (pkAesMode) { - const UInt32 padSize = _pkAesDecoderSpec->GetPadSize((UInt32)processed); + const UInt32 padSize = _pkAesDecoder->GetPadSize((UInt32)processed); if (processed + padSize > coderPackSize) truncatedError = true; else if (processed + padSize < coderPackSize) @@ -1514,7 +1477,7 @@ HRESULT CZipDecoder::Decode( } if (result == S_OK && id == NFileHeader::NCompressionMethod::kLZMA) - if (!lzmaDecoderSpec->DecoderSpec->CheckFinishStatus(item.IsLzmaEOS())) + if (!lzmaDecoderSpec->Decoder->CheckFinishStatus(item.IsLzmaEOS())) lzmaEosError = true; } @@ -1533,32 +1496,32 @@ HRESULT CZipDecoder::Decode( bool crcOK = true; bool authOk = true; if (needCRC) - crcOK = (outStreamSpec->GetCRC() == item.Crc); + crcOK = (outStream->GetCRC() == item.Crc); if (useUnpackLimit) - if (outStreamSpec->GetSize() != item.Size) + if (outStream->GetSize() != item.Size) truncatedError = true; if (wzAesMode) { - const UInt64 unpackSize = outStreamSpec->GetSize(); - const UInt64 packSize = limitedStreamSpec->GetSize(); + const UInt64 unpackSize = outStream->GetSize(); + const UInt64 packSize = inStream->GetSize(); bool thereAreData = false; // read to the end from filter or from packed stream if (SkipStreamData(readFromFilter ? - (ISequentialInStream *)filterStream : - (ISequentialInStream *)inStream, + filterStream.Interface() : + inStream.Interface(), compressProgress, packSize, unpackSize, thereAreData) != S_OK) authOk = false; if (needReminderCheck && thereAreData) dataAfterEnd = true; - if (limitedStreamSpec->GetRem() != 0) + if (inStream->GetRem() != 0) truncatedError = true; else { - limitedStreamSpec->Init(NCrypto::NWzAes::kMacSize); - if (_wzAesDecoderSpec->CheckMac(inStream, authOk) != S_OK) + inStream->Init(NCrypto::NWzAes::kMacSize); + if (_wzAesDecoder->CheckMac(inStream, authOk) != S_OK) authOk = false; } } @@ -1617,8 +1580,7 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, CZipDecoder myDecoder; UInt64 cur_Unpacked, cur_Packed; - CLocalProgress *lps = new CLocalProgress; - CMyComPtr progress = lps; + CMyComPtr2_Create lps; lps->Init(extractCallback, false); for (i = 0;; i++, @@ -1640,6 +1602,8 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, NExtract::NAskMode::kTest : NExtract::NAskMode::kExtract; + Int32 opRes; + { CMyComPtr realOutStream; RINOK(extractCallback->GetStream(index, &realOutStream, askMode)) @@ -1689,25 +1653,24 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, RINOK(extractCallback->PrepareOperation(askMode)) - Int32 res; const HRESULT hres = myDecoder.Decode( EXTERNAL_CODECS_VARS m_Archive, item, realOutStream, extractCallback, - progress, + lps, #ifndef Z7_ST _props._numThreads, _props._memUsage_Decompress, #endif - res); + opRes); RINOK(hres) - realOutStream.Release(); + // realOutStream.Release(); - if (res == NExtract::NOperationResult::kOK && headersError) - res = NExtract::NOperationResult::kHeadersError; - - RINOK(extractCallback->SetOperationResult(res)) + if (opRes == NExtract::NOperationResult::kOK && headersError) + opRes = NExtract::NOperationResult::kHeadersError; + } + RINOK(extractCallback->SetOperationResult(opRes)) } - + COM_TRY_END } diff --git a/CPP/7zip/Archive/Zip/ZipHandlerOut.cpp b/CPP/7zip/Archive/Zip/ZipHandlerOut.cpp index 19699b5..46ccf3d 100644 --- a/CPP/7zip/Archive/Zip/ZipHandlerOut.cpp +++ b/CPP/7zip/Archive/Zip/ZipHandlerOut.cpp @@ -515,17 +515,25 @@ Z7_COM7F_IMF(CHandler::SetProperties(const wchar_t * const *names, const PROPVAR return E_INVALIDARG; { const wchar_t *m = prop.bstrVal; - if (IsString1PrefixedByString2_NoCase_Ascii(m, "aes")) + if (IsString1PrefixedByString2_NoCase_Ascii(m, "AES")) { m += 3; - if (StringsAreEqual_Ascii(m, "128")) - _props.AesKeyMode = 1; - else if (StringsAreEqual_Ascii(m, "192")) - _props.AesKeyMode = 2; - else if (StringsAreEqual_Ascii(m, "256") || m[0] == 0) - _props.AesKeyMode = 3; - else - return E_INVALIDARG; + UInt32 v = 3; + if (*m != 0) + { + if (*m == '-') + m++; + const wchar_t *end; + v = ConvertStringToUInt32(m, &end); + if (*end != 0 || v % 64 != 0) + return E_INVALIDARG; + v /= 64; + v -= 2; + if (v >= 3) + return E_INVALIDARG; + v++; + } + _props.AesKeyMode = (Byte)v; _props.IsAesMode = true; m_ForceAesMode = true; } diff --git a/CPP/7zip/Archive/Zip/ZipIn.cpp b/CPP/7zip/Archive/Zip/ZipIn.cpp index 4236df7..788810f 100644 --- a/CPP/7zip/Archive/Zip/ZipIn.cpp +++ b/CPP/7zip/Archive/Zip/ZipIn.cpp @@ -2282,10 +2282,10 @@ HRESULT CInArchive::ReadVols2(IArchiveOpenVolumeCallback *volCallback, { UString volName = Vols.BaseName; { - volName += (char)(Vols.IsUpperCase ? 'Z' : 'z'); - unsigned v = i + 1; + volName.Add_Char(Vols.IsUpperCase ? 'Z' : 'z'); + const unsigned v = i + 1; if (v < 10) - volName += '0'; + volName.Add_Char('0'); volName.Add_UInt32(v); } diff --git a/CPP/7zip/Archive/Zip/ZipItem.cpp b/CPP/7zip/Archive/Zip/ZipItem.cpp index a77643b..5684cac 100644 --- a/CPP/7zip/Archive/Zip/ZipItem.cpp +++ b/CPP/7zip/Archive/Zip/ZipItem.cpp @@ -55,15 +55,15 @@ void CExtraSubBlock::PrintInfo(AString &s) const { if (Data.Size() >= 1) { - s += ':'; + s.Add_Colon(); const Byte flags = Data[0]; - if (flags & 1) s += 'M'; - if (flags & 2) s += 'A'; - if (flags & 4) s += 'C'; + if (flags & 1) s.Add_Char('M'); + if (flags & 2) s.Add_Char('A'); + if (flags & 4) s.Add_Char('C'); const UInt32 size = (UInt32)(Data.Size()) - 1; if (size % 4 == 0) { - s += ':'; + s.Add_Colon(); s.Add_UInt32(size / 4); } } @@ -88,7 +88,7 @@ void CExtraSubBlock::PrintInfo(AString &s) const } } { - char sz[32]; + char sz[16]; sz[0] = '0'; sz[1] = 'x'; ConvertUInt32ToHex(ID, sz + 2); @@ -291,6 +291,7 @@ bool CItem::IsDir() const case NHostOS::kHPFS: case NHostOS::kVFAT: return true; + default: break; } } @@ -360,6 +361,7 @@ UInt32 CItem::GetWinAttrib() const // #endif } break; + default: break; } if (IsDir()) // test it; winAttrib |= FILE_ATTRIBUTE_DIRECTORY; diff --git a/CPP/7zip/Archive/Zip/ZipUpdate.cpp b/CPP/7zip/Archive/Zip/ZipUpdate.cpp index e6d881c..b2742b7 100644 --- a/CPP/7zip/Archive/Zip/ZipUpdate.cpp +++ b/CPP/7zip/Archive/Zip/ZipUpdate.cpp @@ -976,7 +976,7 @@ static HRESULT Update2( complexity = 0; - const Byte method = options.MethodSequence.Front(); + const Byte method = options.MethodSequence.FrontItem(); COneMethodInfo *oneMethodMain = NULL; if (!options2._methods.IsEmpty()) @@ -1310,7 +1310,7 @@ static HRESULT Update2( { NWindows::NSynchronization::CCriticalSectionLock lock(mtProgressMixerSpec->Mixer2->CriticalSection); - HRESULT res = updateCallback->GetStream(ui.IndexInClient, &fileInStream); + const HRESULT res = updateCallback->GetStream(ui.IndexInClient, &fileInStream); if (res == S_FALSE) { complexity += ui.Size; @@ -1395,7 +1395,7 @@ static HRESULT Update2( if (ui.NewData) { // bool isDir = ((ui.NewProps) ? ui.IsDir : item.IsDir()); - bool isDir = ui.IsDir; + const bool isDir = ui.IsDir; if (isDir) { @@ -1471,7 +1471,7 @@ static HRESULT Update2( } { - CThreadInfo &thread = threads.Threads[threadIndices.Front()]; + CThreadInfo &thread = threads.Threads[threadIndices.FrontItem()]; if (!thread.OutStreamSpec->WasUnlockEventSent()) { CMyComPtr outStream; @@ -1481,11 +1481,11 @@ static HRESULT Update2( } } - WRes wres = mtSem.Semaphore.Lock(); + const WRes wres = mtSem.Semaphore.Lock(); if (wres != 0) return HRESULT_FROM_WIN32(wres); - int ti = mtSem.GetFreeItem(); + const int ti = mtSem.GetFreeItem(); if (ti < 0) return E_FAIL; @@ -1617,9 +1617,9 @@ Z7_COM7F_IMF(CSeekOutStream::SetSize(UInt64 newSize)) } */ -static const size_t kCacheBlockSize = (1 << 20); -static const size_t kCacheSize = (kCacheBlockSize << 2); -static const size_t kCacheMask = (kCacheSize - 1); +static const size_t kCacheBlockSize = 1 << 20; +static const size_t kCacheSize = kCacheBlockSize << 2; +static const size_t kCacheMask = kCacheSize - 1; Z7_CLASS_IMP_NOQIB_2( CCacheOutStream @@ -1628,55 +1628,66 @@ Z7_CLASS_IMP_NOQIB_2( ) Z7_IFACE_COM7_IMP(ISequentialOutStream) - CMyComPtr _stream; + HRESULT _hres; CMyComPtr _seqStream; + CMyComPtr _stream; + CMyComPtr _setRestriction; Byte *_cache; + size_t _cachedSize; + UInt64 _cachedPos; UInt64 _virtPos; UInt64 _virtSize; UInt64 _phyPos; - UInt64 _phySize; // <= _virtSize - UInt64 _cachedPos; // (_cachedPos + _cachedSize) <= _virtSize - size_t _cachedSize; - HRESULT _hres; - + UInt64 _phySize; UInt64 _restrict_begin; UInt64 _restrict_end; - UInt64 _restrict_phy; // begin - CMyComPtr _setRestriction; - HRESULT MyWrite(size_t size); - HRESULT MyWriteBlock() + HRESULT FlushFromCache(size_t size); + HRESULT FlushNonRestrictedBlocks(); + HRESULT FlushCache(); + HRESULT SetRestriction_ForWrite(size_t writeSize) const; + + HRESULT SeekPhy(UInt64 pos) { - return MyWrite(kCacheBlockSize - ((size_t)_cachedPos & (kCacheBlockSize - 1))); + if (pos == _phyPos) + return S_OK; + if (!_stream) + return E_NOTIMPL; + _hres = _stream->Seek((Int64)pos, STREAM_SEEK_SET, &_phyPos); + if (_hres == S_OK && _phyPos != pos) + _hres = E_FAIL; + return _hres; } - HRESULT WriteNonRestrictedBlocks(); - HRESULT FlushCache(); + public: CCacheOutStream(): _cache(NULL) {} ~CCacheOutStream(); - bool Allocate(); + bool Allocate() + { + if (!_cache) + _cache = (Byte *)::MidAlloc(kCacheSize); + return _cache != NULL; + } HRESULT Init(ISequentialOutStream *seqStream, IOutStream *stream, IStreamSetRestriction *setRestriction); HRESULT FinalFlush(); }; - -bool CCacheOutStream::Allocate() +CCacheOutStream::~CCacheOutStream() { - if (!_cache) - _cache = (Byte *)::MidAlloc(kCacheSize); - return (_cache != NULL); + ::MidFree(_cache); } + HRESULT CCacheOutStream::Init(ISequentialOutStream *seqStream, IOutStream *stream, IStreamSetRestriction *setRestriction) { - _cachedPos = 0; - _cachedSize = 0; _hres = S_OK; - _restrict_begin = 0; - _restrict_end = 0; - _restrict_phy = 0; + _cachedSize = 0; + _cachedPos = 0; _virtPos = 0; _virtSize = 0; + // by default we have no restriction + _restrict_begin = 0; + _restrict_end = 0; _seqStream = seqStream; _stream = stream; _setRestriction = setRestriction; @@ -1692,63 +1703,99 @@ HRESULT CCacheOutStream::Init(ISequentialOutStream *seqStream, IOutStream *strea } -/* it writes up to (size) bytes from cache. - (size > _cachedSize) is allowed */ +/* we call SetRestriction_ForWrite() just before Write() from cache. + (_phyPos == _cachedPos) + (writeSize != 0) +*/ +HRESULT CCacheOutStream::SetRestriction_ForWrite(size_t writeSize) const +{ + if (!_setRestriction) + return S_OK; + PRF(printf("\n-- CCacheOutStream::SetRestriction_ForWrite _cachedPos = 0x%x, writeSize = %d\n", (unsigned)_cachedPos, (unsigned)writeSize)); + UInt64 begin = _restrict_begin; + UInt64 end = _restrict_end; + const UInt64 phyPos = _phyPos; + if (phyPos != _cachedPos) return E_FAIL; + if (phyPos == _phySize) + { + // The writing will be to the end of phy stream. + // So we will try to use non-restricted write, if possible. + if (begin == end) + begin = _virtPos; // _virtSize; // it's supposed that (_virtSize == _virtPos) + if (phyPos + writeSize <= begin) + { + // the write is not restricted + PRF(printf("\n+++ write is not restricted \n")); + begin = 0; + end = 0; + } + else + { + if (begin > phyPos) + begin = phyPos; + end = (UInt64)(Int64)-1; + } + } + else + { + // (phyPos != _phySize) + if (begin == end || begin > phyPos) + begin = phyPos; + end = (UInt64)(Int64)-1; + } + return _setRestriction->SetRestriction(begin, end); +} + -HRESULT CCacheOutStream::MyWrite(size_t size) +/* it writes up to (size) bytes from cache. + (size > _cachedSize) is allowed +*/ +HRESULT CCacheOutStream::FlushFromCache(size_t size) { - PRF(printf("\n-- CCacheOutStream::MyWrite %u\n", (unsigned)size)); + PRF(printf("\n-- CCacheOutStream::FlushFromCache %u\n", (unsigned)size)); if (_hres != S_OK) return _hres; - while (size != 0 && _cachedSize != 0) + if (size == 0 || _cachedSize == 0) + return S_OK; + RINOK(SeekPhy(_cachedPos)) + for (;;) { - if (_phyPos != _cachedPos) - { - if (!_stream) - return E_FAIL; - _hres = _stream->Seek((Int64)_cachedPos, STREAM_SEEK_SET, &_phyPos); - RINOK(_hres) - if (_phyPos != _cachedPos) - { - _hres = E_FAIL; - return _hres; - } - } + // (_phyPos == _cachedPos) const size_t pos = (size_t)_cachedPos & kCacheMask; - size_t curSize = kCacheSize - pos; - curSize = MyMin(curSize, _cachedSize); - curSize = MyMin(curSize, size); - _hres = WriteStream(_seqStream, _cache + pos, curSize); + size_t cur = kCacheSize - pos; + cur = MyMin(cur, _cachedSize); + cur = MyMin(cur, size); + _hres = SetRestriction_ForWrite(cur); + RINOK(_hres) + PRF(printf("\n-- CCacheOutStream::WriteFromCache _phyPos = 0x%x, size = %d\n", (unsigned)_phyPos, (unsigned)cur)); + _hres = WriteStream(_seqStream, _cache + pos, cur); RINOK(_hres) - _phyPos += curSize; + _phyPos += cur; if (_phySize < _phyPos) _phySize = _phyPos; - _cachedPos += curSize; - _cachedSize -= curSize; - size -= curSize; + _cachedPos += cur; + _cachedSize -= cur; + size -= cur; + if (size == 0 || _cachedSize == 0) + return S_OK; } - - if (_setRestriction) - if (_restrict_begin == _restrict_end || _cachedPos <= _restrict_begin) - if (_restrict_phy < _cachedPos) - { - _restrict_phy = _cachedPos; - return _setRestriction->SetRestriction(_cachedPos, (UInt64)(Int64)-1); - } - return S_OK; } -HRESULT CCacheOutStream::WriteNonRestrictedBlocks() +HRESULT CCacheOutStream::FlushNonRestrictedBlocks() { for (;;) { const size_t size = kCacheBlockSize - ((size_t)_cachedPos & (kCacheBlockSize - 1)); if (_cachedSize < size) break; - if (_restrict_begin != _restrict_end && _cachedPos + size > _restrict_begin) + UInt64 begin = _restrict_begin; + if (begin == _restrict_end) + begin = _virtPos; + // we don't flush the data to restricted area + if (_cachedPos + size > begin) break; - RINOK(MyWrite(size)) + RINOK(FlushFromCache(size)) } return S_OK; } @@ -1756,7 +1803,7 @@ HRESULT CCacheOutStream::WriteNonRestrictedBlocks() HRESULT CCacheOutStream::FlushCache() { - return MyWrite(_cachedSize); + return FlushFromCache(_cachedSize); } HRESULT CCacheOutStream::FinalFlush() @@ -1770,25 +1817,17 @@ HRESULT CCacheOutStream::FinalFlush() { // it's unexpected RINOK(_stream->SetSize(_virtSize)) + _phySize = _virtSize; } - if (_virtPos != _phyPos) - { - RINOK(_stream->Seek((Int64)_virtPos, STREAM_SEEK_SET, NULL)) - } + _hres = SeekPhy(_virtPos); } - return S_OK; -} - - -CCacheOutStream::~CCacheOutStream() -{ - ::MidFree(_cache); + return _hres; } Z7_COM7F_IMF(CCacheOutStream::Write(const void *data, UInt32 size, UInt32 *processedSize)) { - PRF(printf("\n-- CCacheOutStream::Write %u\n", (unsigned)size)); + PRF(printf("\n==== CCacheOutStream::Write virtPos=0x%x, %u\n", (unsigned)_virtPos, (unsigned)size)); if (processedSize) *processedSize = 0; @@ -1804,38 +1843,74 @@ Z7_COM7F_IMF(CCacheOutStream::Write(const void *data, UInt32 size, UInt32 *proce RINOK(FlushCache()) } - // ---------- Writing data to cache ---------- - if (_cachedSize == 0) _cachedPos = _virtPos; const size_t pos = (size_t)_virtPos & kCacheMask; - size = (UInt32)MyMin((size_t)size, kCacheSize - pos); - const UInt64 cachedEnd = _cachedPos + _cachedSize; - - // (_virtPos >= _cachedPos) (_virtPos <= cachedEnd) - - if (_virtPos != cachedEnd) { - // _virtPos < cachedEnd + const size_t blockRem = kCacheBlockSize - ((size_t)_virtPos & (kCacheBlockSize - 1)); + if (size > blockRem) + size = (UInt32)blockRem; + } + // _cachedPos <= _virtPos <= _cachedPos + _cachedSize + const UInt64 cachedRem = _cachedPos + _cachedSize - _virtPos; + if (cachedRem) + { + // _virtPos < _cachedPos + _cachedSize // we rewrite only existing data in cache. So _cachedSize will be not changed - size = (UInt32)MyMin((size_t)size, (size_t)(cachedEnd - _virtPos)); + if (size > cachedRem) + size = (UInt32)cachedRem; } else { - // _virtPos == cachedEnd + // _virtPos == _cachedPos + _cachedSize // so we need to add new data to the end of cache if (_cachedSize == kCacheSize) { - // cache is full. So we flush part of cache - RINOK(MyWriteBlock()) + // cache is full. So we need to flush some part of cache. + // we flush only one block, but we are allowed to flush any size here + RINOK(FlushFromCache(kCacheBlockSize - ((size_t)_cachedPos & (kCacheBlockSize - 1)))) } // _cachedSize != kCacheSize // so we have some space for new data in cache - const size_t startPos = (size_t)_cachedPos & kCacheMask; - // we don't allow new data to overwrite old start data in cache. - if (startPos > pos) - size = (UInt32)MyMin((size_t)size, (size_t)(startPos - pos)); + if (_cachedSize == 0) + { + /* this code is optional (for optimization): + we write data directly without cache, + if there is no restriction and we have full block. */ + if (_restrict_begin == _restrict_end + && size == kCacheBlockSize) + { + RINOK(SeekPhy(_virtPos)) + if (_setRestriction) + { + _hres = _setRestriction->SetRestriction(_restrict_begin, _restrict_end); + RINOK(_hres) + } + PRF(printf("\n-- CCacheOutStream::WriteDirectly _phyPos = 0x%x, size = %d\n", (unsigned)_phyPos, (unsigned)size)); + _hres = WriteStream(_seqStream, data, size); + RINOK(_hres) + if (processedSize) + *processedSize = size; + _virtPos += size; + if (_virtSize < _virtPos) + _virtSize = _virtPos; + _phyPos += size; + if (_phySize < _phyPos) + _phySize = _phyPos; + return S_OK; + } + } + else // (_cachedSize != 0) + { + const size_t startPos = (size_t)_cachedPos & kCacheMask; + // we don't allow new data to overwrite old start data in cache. + // (startPos == pos) here means that cache is empty. + // (startPos == pos) is not possible here. + if (startPos > pos) + size = (UInt32)MyMin((size_t)size, (size_t)(startPos - pos)); + } + // _virtPos == (_cachedPos + _cachedSize) still _cachedSize += size; } @@ -1845,13 +1920,13 @@ Z7_COM7F_IMF(CCacheOutStream::Write(const void *data, UInt32 size, UInt32 *proce _virtPos += size; if (_virtSize < _virtPos) _virtSize = _virtPos; - return WriteNonRestrictedBlocks(); + return FlushNonRestrictedBlocks(); } Z7_COM7F_IMF(CCacheOutStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *newPosition)) { - PRF(printf("\n-- CCacheOutStream::Seek seekOrigin=%d Seek =%u\n", seekOrigin, (unsigned)offset)); + PRF(printf("\n==== CCacheOutStream::Seek seekOrigin=%d Seek =%u\n", seekOrigin, (unsigned)offset)); switch (seekOrigin) { case STREAM_SEEK_SET: break; @@ -1872,35 +1947,63 @@ Z7_COM7F_IMF(CCacheOutStream::SetSize(UInt64 newSize)) { if (_hres != S_OK) return _hres; - _virtSize = newSize; - - if (newSize <= _cachedPos) + + if (newSize <= _cachedPos || _cachedSize == 0) { _cachedSize = 0; _cachedPos = newSize; } else { + // _cachedSize != 0 // newSize > _cachedPos const UInt64 offset = newSize - _cachedPos; if (offset <= _cachedSize) { + // newSize is inside cached block or is touching cached block. + // so we reduce cache _cachedSize = (size_t)offset; if (_phySize <= newSize) - return S_OK; + return S_OK; // _phySize will be restored later after cache flush + // (_phySize > newSize) + // so we must reduce phyStream size to (newSize) or to (_cachedPos) + // newPhySize = _cachedPos; // optional reduce to _cachedPos } else { // newSize > _cachedPos + _cachedSize - // So we flush cache - RINOK(FlushCache()) + /* It's possible that we need to write zeros, + if new size is larger than old size. + We don't optimize for possible cases here. + So we just flush the cache. */ + _hres = FlushCache(); } } + _virtSize = newSize; + + if (_hres != S_OK) + return _hres; + if (newSize != _phySize) { if (!_stream) return E_NOTIMPL; + // if (_phyPos > newSize) + RINOK(SeekPhy(newSize)) + if (_setRestriction) + { + UInt64 begin = _restrict_begin; + UInt64 end = _restrict_end; + if (_cachedSize != 0) + { + if (begin > _cachedPos) + begin = _cachedPos; + end = (UInt64)(Int64)-1; + } + _hres = _setRestriction->SetRestriction(begin, end); + RINOK(_hres) + } _hres = _stream->SetSize(newSize); RINOK(_hres) _phySize = newSize; @@ -1911,10 +2014,10 @@ Z7_COM7F_IMF(CCacheOutStream::SetSize(UInt64 newSize)) Z7_COM7F_IMF(CCacheOutStream::SetRestriction(UInt64 begin, UInt64 end)) { - PRF(printf("\n============ CCacheOutStream::SetRestriction %u, %u\n", (unsigned)begin, (unsigned)end)); + PRF(printf("\n============ CCacheOutStream::SetRestriction 0x%x, %u\n", (unsigned)begin, (unsigned)end)); _restrict_begin = begin; _restrict_end = end; - return WriteNonRestrictedBlocks(); + return FlushNonRestrictedBlocks(); } @@ -1971,6 +2074,11 @@ HRESULT Update( } } + outSeqMode = (outStreamReal == NULL); + if (outSeqMode) + setRestriction.Release(); + /* CCacheOutStream works as non-restricted by default. + So we use (setRestriction == NULL) for outSeqMode */ // bool use_cacheStream = true; // if (use_cacheStream) { @@ -1980,7 +2088,8 @@ HRESULT Update( return E_OUTOFMEMORY; RINOK(cacheStream->Init(seqOutStream, outStreamReal, setRestriction)) setRestriction.Release(); - setRestriction = cacheStream; + if (!outSeqMode) + setRestriction = cacheStream; } /* else if (!outStreamReal) @@ -1992,7 +2101,6 @@ HRESULT Update( else outStream = outStreamReal; */ - outSeqMode = (outStreamReal == NULL); } COutArchive outArchive; diff --git a/CPP/7zip/Archive/ZstdHandler.cpp b/CPP/7zip/Archive/ZstdHandler.cpp new file mode 100644 index 0000000..b9c0fb7 --- /dev/null +++ b/CPP/7zip/Archive/ZstdHandler.cpp @@ -0,0 +1,1147 @@ +// ZstdHandler.cpp + +#include "StdAfx.h" + +// #define Z7_USE_ZSTD_ORIG_DECODER +// #define Z7_USE_ZSTD_COMPRESSION + +#include "../../Common/ComTry.h" + +#include "../Common/MethodProps.h" +#include "../Common/ProgressUtils.h" +#include "../Common/RegisterArc.h" +#include "../Common/StreamUtils.h" + +#include "../Compress/CopyCoder.h" +#include "../Compress/ZstdDecoder.h" +#ifdef Z7_USE_ZSTD_ORIG_DECODER +#include "../Compress/Zstd2Decoder.h" +#endif + +#ifdef Z7_USE_ZSTD_COMPRESSION +#include "../Compress/ZstdEncoder.h" +#include "../Compress/ZstdEncoderProps.h" +#include "Common/HandlerOut.h" +#endif + +#include "Common/DummyOutStream.h" + +#include "../../../C/CpuArch.h" + +using namespace NWindows; + +namespace NArchive { +namespace NZstd { + +#define DESCRIPTOR_Get_DictionaryId_Flag(d) ((d) & 3) +#define DESCRIPTOR_FLAG_CHECKSUM (1 << 2) +#define DESCRIPTOR_FLAG_RESERVED (1 << 3) +#define DESCRIPTOR_FLAG_UNUSED (1 << 4) +#define DESCRIPTOR_FLAG_SINGLE (1 << 5) +#define DESCRIPTOR_Get_ContentSize_Flag3(d) ((d) >> 5) +#define DESCRIPTOR_Is_ContentSize_Defined(d) (((d) & 0xe0) != 0) + +struct CFrameHeader +{ + Byte Descriptor; + Byte WindowDescriptor; + UInt32 DictionaryId; + UInt64 ContentSize; + + /* by zstd specification: + the decoder must check that (Is_Reserved() == false) + the decoder must ignore Unused_bit */ + bool Is_Reserved() const { return (Descriptor & DESCRIPTOR_FLAG_RESERVED) != 0; } + bool Is_Checksum() const { return (Descriptor & DESCRIPTOR_FLAG_CHECKSUM) != 0; } + bool Is_SingleSegment() const { return (Descriptor & DESCRIPTOR_FLAG_SINGLE) != 0; } + bool Is_ContentSize_Defined() const { return DESCRIPTOR_Is_ContentSize_Defined(Descriptor); } + unsigned Get_DictionaryId_Flag() const { return DESCRIPTOR_Get_DictionaryId_Flag(Descriptor); } + unsigned Get_ContentSize_Flag3() const { return DESCRIPTOR_Get_ContentSize_Flag3(Descriptor); } + + const Byte *Parse(const Byte *p, int size) + { + if ((unsigned)size < 2) + return NULL; + Descriptor = *p++; + size--; + { + Byte w = 0; + if (!Is_SingleSegment()) + { + w = *p++; + size--; + } + WindowDescriptor = w; + } + { + unsigned n = Get_DictionaryId_Flag(); + UInt32 d = 0; + if (n) + { + n = (unsigned)1 << (n - 1); + if ((size -= (int)n) < 0) + return NULL; + d = GetUi32(p) & ((UInt32)(Int32)-1 >> (32 - 8u * n)); + p += n; + } + DictionaryId = d; + } + { + unsigned n = Get_ContentSize_Flag3(); + UInt64 v = 0; + if (n) + { + n >>= 1; + if (n == 1) + v = 256; + n = (unsigned)1 << n; + if ((size -= (int)n) < 0) + return NULL; + v += GetUi64(p) & ((UInt64)(Int64)-1 >> (64 - 8u * n)); + p += n; + } + ContentSize = v; + } + return p; + } +}; + + + +class CHandler Z7_final: + public IInArchive, + public IArchiveOpenSeq, + public ISetProperties, +#ifdef Z7_USE_ZSTD_COMPRESSION + public IOutArchive, +#endif + public CMyUnknownImp +{ + Z7_COM_QI_BEGIN2(IInArchive) + Z7_COM_QI_ENTRY(IArchiveOpenSeq) + Z7_COM_QI_ENTRY(ISetProperties) +#ifdef Z7_USE_ZSTD_COMPRESSION + Z7_COM_QI_ENTRY(IOutArchive) +#endif + Z7_COM_QI_END + Z7_COM_ADDREF_RELEASE + + Z7_IFACE_COM7_IMP(IInArchive) + Z7_IFACE_COM7_IMP(IArchiveOpenSeq) + Z7_IFACE_COM7_IMP(ISetProperties) +#ifdef Z7_USE_ZSTD_COMPRESSION + Z7_IFACE_COM7_IMP(IOutArchive) +#endif + + bool _isArc; + bool _needSeekToStart; + // bool _dataAfterEnd; + // bool _needMoreInput; + bool _unsupportedBlock; + + bool _wasParsed; + bool _phySize_Decoded_Defined; + bool _unpackSize_Defined; // decoded + bool _decoded_Info_Defined; + + bool _parseMode; + bool _disableHash; + // bool _smallMode; + + UInt64 _phySize; + UInt64 _phySize_Decoded; + UInt64 _unpackSize; + + CZstdDecInfo _parsed_Info; + CZstdDecInfo _decoded_Info; + + CMyComPtr _stream; + CMyComPtr _seqStream; + +#ifdef Z7_USE_ZSTD_COMPRESSION + CSingleMethodProps _props; +#endif + +public: + CHandler(): + _parseMode(false), + _disableHash(false) + // _smallMode(false) + {} +}; + + +static const Byte kProps[] = +{ + kpidSize, + kpidPackSize +}; + +static const Byte kArcProps[] = +{ + kpidNumStreams, + kpidNumBlocks, + kpidMethod, + // kpidChecksum + kpidCRC +}; + +IMP_IInArchive_Props +IMP_IInArchive_ArcProps + + +// static const unsigned kBlockType_Raw = 0; +static const unsigned kBlockType_RLE = 1; +// static const unsigned kBlockType_Compressed = 2; +static const unsigned kBlockType_Reserved = 3; +/* +static const char * const kNames[] = +{ + "RAW" + , "RLE" + , "Compressed" + , "Reserved" +}; +*/ + +static void Add_UInt64(AString &s, const char *name, UInt64 v) +{ + s.Add_OptSpaced(name); + s.Add_Colon(); + s.Add_UInt64(v); +} + + +static void PrintSize(AString &s, UInt64 w) +{ + char c = 0; + if ((w & ((1 << 30) - 1)) == 0) { c = 'G'; w >>= 30; } + else if ((w & ((1 << 20) - 1)) == 0) { c = 'M'; w >>= 20; } + else if ((w & ((1 << 10) - 1)) == 0) { c = 'K'; w >>= 10; } + s.Add_UInt64(w); + if (c) + { + s.Add_Char(c); + s += "iB"; + } +} + + +Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) +{ + NCOM::CPropVariant prop; + + CZstdDecInfo *p = NULL; + if (_wasParsed || !_decoded_Info_Defined) + p = &_parsed_Info; + else if (_decoded_Info_Defined) + p = &_decoded_Info; + + switch (propID) + { + case kpidPhySize: + if (_wasParsed) + prop = _phySize; + else if (_phySize_Decoded_Defined) + prop = _phySize_Decoded; + break; + + case kpidUnpackSize: + if (_unpackSize_Defined) + prop = _unpackSize; + break; + + case kpidNumStreams: + if (p) + if (_wasParsed || _decoded_Info_Defined) + prop = p->num_DataFrames; + break; + + case kpidNumBlocks: + if (p) + if (_wasParsed || _decoded_Info_Defined) + prop = p->num_Blocks; + break; + + // case kpidChecksum: + case kpidCRC: + if (p) + if (p->checksum_Defined && p->num_DataFrames == 1) + prop = p->checksum; // it's checksum from last frame + break; + + case kpidMethod: + { + AString s; + s.Add_OptSpaced(p == &_decoded_Info ? + "decoded:" : _wasParsed ? + "parsed:" : + "header-open-only:"); + + if (p->dictionaryId != 0) + { + if (p->are_DictionaryId_Different) + s.Add_OptSpaced("different-dictionary-IDs"); + s.Add_OptSpaced("dictionary-ID:"); + s.Add_UInt32(p->dictionaryId); + } + /* + if (ContentSize_Defined) + { + s.Add_OptSpaced("ContentSize="); + s.Add_UInt64(ContentSize_Total); + } + */ + // if (p->are_Checksums) + if (p->descriptor_OR & DESCRIPTOR_FLAG_CHECKSUM) + s.Add_OptSpaced("XXH64"); + if (p->descriptor_NOT_OR & DESCRIPTOR_FLAG_CHECKSUM) + s.Add_OptSpaced("NO-XXH64"); + + if (p->descriptor_OR & DESCRIPTOR_FLAG_UNUSED) + s.Add_OptSpaced("unused_bit"); + + if (p->descriptor_OR & DESCRIPTOR_FLAG_SINGLE) + s.Add_OptSpaced("single-segments"); + + if (p->descriptor_NOT_OR & DESCRIPTOR_FLAG_SINGLE) + { + // Add_UInt64(s, "wnd-descriptors", p->num_WindowDescriptors); + s.Add_OptSpaced("wnd-desc-log-MAX:"); + // WindowDescriptor_MAX = 16 << 3; // for debug + const unsigned e = p->windowDescriptor_MAX >> 3; + s.Add_UInt32(e + 10); + const unsigned m = p->windowDescriptor_MAX & 7; + if (m != 0) + { + s.Add_Dot(); + s.Add_UInt32(m); + } + } + + if (DESCRIPTOR_Is_ContentSize_Defined(p->descriptor_OR) || + (p->descriptor_NOT_OR & DESCRIPTOR_FLAG_SINGLE)) + /* + if (p->are_ContentSize_Known || + p->are_WindowDescriptors) + */ + { + s.Add_OptSpaced("wnd-MAX:"); + PrintSize(s, p->windowSize_MAX); + if (p->windowSize_MAX != p->windowSize_Allocate_MAX) + { + s.Add_OptSpaced("wnd-use-MAX:"); + PrintSize(s, p->windowSize_Allocate_MAX); + } + } + + if (p->num_DataFrames != 1) + Add_UInt64(s, "data-frames", p->num_DataFrames); + if (p->num_SkipFrames != 0) + { + Add_UInt64(s, "skip-frames", p->num_SkipFrames); + Add_UInt64(s, "skip-frames-size-total", p->skipFrames_Size); + } + + if (p->are_ContentSize_Unknown) + s.Add_OptSpaced("unknown-content-size"); + + if (DESCRIPTOR_Is_ContentSize_Defined(p->descriptor_OR)) + { + Add_UInt64(s, "content-size-frame-max", p->contentSize_MAX); + Add_UInt64(s, "content-size-total", p->contentSize_Total); + } + + /* + for (unsigned i = 0; i < 4; i++) + { + const UInt64 n = p->num_Blocks_forType[i]; + if (n) + { + s.Add_OptSpaced(kNames[i]); + s += "-blocks:"; + s.Add_UInt64(n); + + s.Add_OptSpaced(kNames[i]); + s += "-block-bytes:"; + s.Add_UInt64(p->num_BlockBytes_forType[i]); + } + } + */ + prop = s; + break; + } + + case kpidErrorFlags: + { + UInt32 v = 0; + if (!_isArc) v |= kpv_ErrorFlags_IsNotArc; + // if (_needMoreInput) v |= kpv_ErrorFlags_UnexpectedEnd; + // if (_dataAfterEnd) v |= kpv_ErrorFlags_DataAfterEnd; + if (_unsupportedBlock) v |= kpv_ErrorFlags_UnsupportedMethod; + /* + if (_parsed_Info.numBlocks_forType[kBlockType_Reserved]) + v |= kpv_ErrorFlags_UnsupportedMethod; + */ + prop = v; + break; + } + + default: break; + } + prop.Detach(value); + return S_OK; +} + + +Z7_COM7F_IMF(CHandler::GetNumberOfItems(UInt32 *numItems)) +{ + *numItems = 1; + return S_OK; +} + +Z7_COM7F_IMF(CHandler::GetProperty(UInt32 /* index */, PROPID propID, PROPVARIANT *value)) +{ + NCOM::CPropVariant prop; + switch (propID) + { + case kpidPackSize: + if (_wasParsed) + prop = _phySize; + else if (_phySize_Decoded_Defined) + prop = _phySize_Decoded; + break; + + case kpidSize: + if (_wasParsed && !_parsed_Info.are_ContentSize_Unknown) + prop = _parsed_Info.contentSize_Total; + else if (_unpackSize_Defined) + prop = _unpackSize; + break; + + default: break; + } + prop.Detach(value); + return S_OK; +} + +static const unsigned kSignatureSize = 4; +static const Byte k_Signature[kSignatureSize] = { 0x28, 0xb5, 0x2f, 0xfd } ; + +static const UInt32 kDataFrameSignature32 = 0xfd2fb528; +static const UInt32 kSkipFrameSignature = 0x184d2a50; +static const UInt32 kSkipFrameSignature_Mask = 0xfffffff0; + +/* +API_FUNC_static_IsArc IsArc_Zstd(const Byte *p, size_t size) +{ + if (size < kSignatureSize) + return k_IsArc_Res_NEED_MORE; + if (memcmp(p, k_Signature, kSignatureSize) != 0) + { + const UInt32 v = GetUi32(p); + if ((v & kSkipFrameSignature_Mask) != kSkipFrameSignature) + return k_IsArc_Res_NO; + return k_IsArc_Res_YES; + } + p += 4; + // return k_IsArc_Res_YES; +} +} +*/ + +// kBufSize must be >= (ZSTD_FRAMEHEADERSIZE_MAX = 18) +// we use big buffer for fast parsing of worst case small blocks. +static const unsigned kBufSize = + 1 << 9; + // 1 << 14; // fastest in real file + +struct CStreamBuffer +{ + unsigned pos; + unsigned lim; + IInStream *Stream; + UInt64 StreamOffset; + Byte buf[kBufSize]; + + CStreamBuffer(): + pos(0), + lim(0), + StreamOffset(0) + {} + unsigned Avail() const { return lim - pos; } + const Byte *GetPtr() const { return &buf[pos]; } + UInt64 GetCurOffset() const { return StreamOffset - Avail(); } + void SkipInBuf(UInt32 size) { pos += size; } + HRESULT Skip(UInt32 size); + HRESULT Read(unsigned num); +}; + +HRESULT CStreamBuffer::Skip(UInt32 size) +{ + unsigned rem = lim - pos; + if (rem != 0) + { + if (rem > size) + rem = size; + pos += rem; + size -= rem; + if (pos != lim) + return S_OK; + } + if (size == 0) + return S_OK; + return Stream->Seek(size, STREAM_SEEK_CUR, &StreamOffset); +} + +HRESULT CStreamBuffer::Read(unsigned num) +{ + if (lim - pos >= num) + return S_OK; + if (pos != 0) + { + lim -= pos; + memmove(buf, buf + pos, lim); + pos = 0; + } + size_t processed = kBufSize - ((unsigned)StreamOffset & (kBufSize - 1)); + const unsigned avail = kBufSize - lim; + num -= lim; + if (avail < processed || processed < num) + processed = avail; + const HRESULT res = ReadStream(Stream, buf + lim, &processed); + StreamOffset += processed; + lim += (unsigned)processed; + return res; +} + + +static const unsigned k_ZSTD_FRAMEHEADERSIZE_MAX = 4 + 14; + +Z7_COM7F_IMF(CHandler::Open(IInStream *stream, const UInt64 *, IArchiveOpenCallback *callback)) +{ + COM_TRY_BEGIN + Close(); + + CZstdDecInfo *p = &_parsed_Info; + // p->are_ContentSize_Unknown = False; + CStreamBuffer sb; + sb.Stream = stream; + + for (;;) + { + RINOK(sb.Read(k_ZSTD_FRAMEHEADERSIZE_MAX)) + if (sb.Avail() < kSignatureSize) + break; + + if (callback && (ZstdDecInfo_GET_NUM_FRAMES(p) & 0xFFF) == 2) + { + const UInt64 numBytes = sb.GetCurOffset(); + RINOK(callback->SetCompleted(NULL, &numBytes)) + } + + const UInt32 v = GetUi32(sb.GetPtr()); + if (v != kDataFrameSignature32) + { + if ((v & kSkipFrameSignature_Mask) != kSkipFrameSignature) + break; + _phySize = sb.GetCurOffset() + 8; + p->num_SkipFrames++; + sb.SkipInBuf(4); + if (sb.Avail() < 4) + break; + const UInt32 size = GetUi32(sb.GetPtr()); + p->skipFrames_Size += size; + sb.SkipInBuf(4); + _phySize = sb.GetCurOffset() + size; + RINOK(sb.Skip(size)) + continue; + } + + p->num_DataFrames++; + // _numStreams_Defined = true; + sb.SkipInBuf(4); + CFrameHeader fh; + { + const Byte *data = fh.Parse(sb.GetPtr(), (int)sb.Avail()); + if (!data) + { + // _needMoreInput = true; + // we set size for one byte more to show that stream was truncated + _phySize = sb.StreamOffset + 1; + break; + } + if (fh.Is_Reserved()) + { + // we don't want false detection + if (ZstdDecInfo_GET_NUM_FRAMES(p) == 1) + return S_FALSE; + // _phySize = sb.GetCurOffset(); + break; + } + sb.SkipInBuf((unsigned)(data - sb.GetPtr())); + } + + p->descriptor_OR = (Byte)(p->descriptor_OR | fh.Descriptor); + p->descriptor_NOT_OR = (Byte)(p->descriptor_NOT_OR | ~fh.Descriptor); + + // _numBlocks_Defined = true; + // if (fh.Get_DictionaryId_Flag()) + // p->dictionaryId_Cur = fh.DictionaryId; + if (fh.DictionaryId != 0) + { + if (p->dictionaryId == 0) + p->dictionaryId = fh.DictionaryId; + else if (p->dictionaryId != fh.DictionaryId) + p->are_DictionaryId_Different = True; + } + + UInt32 blockSizeAllowedMax = (UInt32)1 << 17; + { + UInt64 winSize = fh.ContentSize; + UInt64 winSize_forAllocate = fh.ContentSize; + if (!fh.Is_SingleSegment()) + { + if (p->windowDescriptor_MAX < fh.WindowDescriptor) + p->windowDescriptor_MAX = fh.WindowDescriptor; + const unsigned e = (fh.WindowDescriptor >> 3); + const unsigned m = (fh.WindowDescriptor & 7); + winSize = (UInt64)(8 + m) << (e + 10 - 3); + if (!fh.Is_ContentSize_Defined() + || fh.DictionaryId != 0 + || winSize_forAllocate > winSize) + winSize_forAllocate = winSize; + // p->are_WindowDescriptors = true; + } + else + { + // p->are_SingleSegments = True; + } + if (blockSizeAllowedMax > winSize) + blockSizeAllowedMax = (UInt32)winSize; + if (p->windowSize_MAX < winSize) + p->windowSize_MAX = winSize; + if (p->windowSize_Allocate_MAX < winSize_forAllocate) + p->windowSize_Allocate_MAX = winSize_forAllocate; + } + + if (fh.Is_ContentSize_Defined()) + { + // p->are_ContentSize_Known = True; + p->contentSize_Total += fh.ContentSize; + if (p->contentSize_MAX < fh.ContentSize) + p->contentSize_MAX = fh.ContentSize; + } + else + { + p->are_ContentSize_Unknown = True; + } + + p->checksum_Defined = false; + + // p->numBlocks_forType[3] += 99; // for debug + + if (!_parseMode) + { + if (ZstdDecInfo_GET_NUM_FRAMES(p) == 1) + break; + } + + _wasParsed = true; + + bool blocksWereParsed = false; + + for (;;) + { + if (callback && (p->num_Blocks & 0xFFF) == 2) + { + // Sleep(10); + const UInt64 numBytes = sb.GetCurOffset(); + RINOK(callback->SetCompleted(NULL, &numBytes)) + } + _phySize = sb.GetCurOffset() + 3; + RINOK(sb.Read(3)) + if (sb.Avail() < 3) + { + // _needMoreInput = true; + // return S_FALSE; + break; // change it + } + const unsigned pos = sb.pos; + sb.pos = pos + 3; + UInt32 b = 0; + b += (UInt32)sb.buf[pos]; + b += (UInt32)sb.buf[pos + 1] << (8 * 1); + b += (UInt32)sb.buf[pos + 2] << (8 * 2); + p->num_Blocks++; + const unsigned blockType = (b >> 1) & 3; + UInt32 size = b >> 3; + // p->num_Blocks_forType[blockType]++; + // p->num_BlockBytes_forType[blockType] += size; + if (size > blockSizeAllowedMax + || blockType == kBlockType_Reserved) + { + _unsupportedBlock = true; + if (ZstdDecInfo_GET_NUM_FRAMES(p) == 1 && p->num_Blocks == 1) + return S_FALSE; + break; + } + if (blockType == kBlockType_RLE) + size = 1; + _phySize = sb.GetCurOffset() + size; + RINOK(sb.Skip(size)) + if (b & 1) + { + // it's last block + blocksWereParsed = true; + break; + } + } + + if (!blocksWereParsed) + break; + + if (fh.Is_Checksum()) + { + _phySize = sb.GetCurOffset() + 4; + RINOK(sb.Read(4)) + if (sb.Avail() < 4) + break; + p->checksum_Defined = true; + // if (p->num_DataFrames == 1) + p->checksum = GetUi32(sb.GetPtr()); + sb.SkipInBuf(4); + } + } + + if (ZstdDecInfo_GET_NUM_FRAMES(p) == 0) + return S_FALSE; + + _needSeekToStart = true; + // } // _parseMode + _isArc = true; + _stream = stream; + _seqStream = stream; + + return S_OK; + COM_TRY_END +} + + +Z7_COM7F_IMF(CHandler::OpenSeq(ISequentialInStream *stream)) +{ + Close(); + _isArc = true; + _seqStream = stream; + return S_OK; +} + + +Z7_COM7F_IMF(CHandler::Close()) +{ + _isArc = false; + _needSeekToStart = false; + // _dataAfterEnd = false; + // _needMoreInput = false; + _unsupportedBlock = false; + + _wasParsed = false; + _phySize_Decoded_Defined = false; + _unpackSize_Defined = false; + _decoded_Info_Defined = false; + + ZstdDecInfo_CLEAR(&_parsed_Info) + ZstdDecInfo_CLEAR(&_decoded_Info) + + _phySize = 0; + _phySize_Decoded = 0; + _unpackSize = 0; + + _seqStream.Release(); + _stream.Release(); + return S_OK; +} + + +Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, + Int32 testMode, IArchiveExtractCallback *extractCallback)) +{ + COM_TRY_BEGIN + if (numItems == 0) + return S_OK; + if (numItems != (UInt32)(Int32)-1 && (numItems != 1 || indices[0] != 0)) + return E_INVALIDARG; + if (_wasParsed) + { + RINOK(extractCallback->SetTotal(_phySize)) + } + + Int32 opRes; + { + CMyComPtr realOutStream; + const Int32 askMode = testMode ? + NExtract::NAskMode::kTest : + NExtract::NAskMode::kExtract; + RINOK(extractCallback->GetStream(0, &realOutStream, askMode)) + if (!testMode && !realOutStream) + return S_OK; + + extractCallback->PrepareOperation(askMode); + + if (_needSeekToStart) + { + if (!_stream) + return E_FAIL; + RINOK(_stream->Seek(0, STREAM_SEEK_SET, NULL)) + } + else + _needSeekToStart = true; + + CMyComPtr2_Create lps; + lps->Init(extractCallback, true); + +#ifdef Z7_USE_ZSTD_ORIG_DECODER + CMyComPtr2_Create decoder; +#else + CMyComPtr2_Create decoder; +#endif + + CMyComPtr2_Create outStreamSpec; + outStreamSpec->SetStream(realOutStream); + outStreamSpec->Init(); + // realOutStream.Release(); + + decoder->FinishMode = true; +#ifndef Z7_USE_ZSTD_ORIG_DECODER + decoder->DisableHash = _disableHash; +#endif + + // _dataAfterEnd = false; + // _needMoreInput = false; + const HRESULT hres = decoder.Interface()->Code(_seqStream, outStreamSpec, NULL, NULL, lps); + /* + { + UInt64 t1 = decoder->GetInputProcessedSize(); + // for debug + const UInt32 kTempSize = 64; + Byte buf[kTempSize]; + UInt32 processedSize = 0; + RINOK(decoder->ReadUnusedFromInBuf(buf, kTempSize, &processedSize)) + processedSize -= processedSize; + UInt64 t2 = decoder->GetInputProcessedSize(); + t2 = t2; + t1 = t1; + } + */ + const UInt64 outSize = outStreamSpec->GetSize(); + // } + + // if (hres == E_ABORT) return hres; + opRes = NExtract::NOperationResult::kDataError; + + if (hres == E_OUTOFMEMORY) + { + return hres; + // opRes = NExtract::NOperationResult::kMemError; + } + else if (hres == S_OK || hres == S_FALSE) + { +#ifndef Z7_USE_ZSTD_ORIG_DECODER + _decoded_Info_Defined = true; + _decoded_Info = decoder->_state.info; + // NumDataFrames_Decoded = decoder->_state.info.num_DataFrames; + // NumSkipFrames_Decoded = decoder->_state.info.num_SkipFrames; + const UInt64 inSize = decoder->_inProcessed; +#else + const UInt64 inSize = decoder->GetInputProcessedSize(); +#endif + _phySize_Decoded = inSize; + _phySize_Decoded_Defined = true; + + _unpackSize_Defined = true; + _unpackSize = outSize; + + // RINOK( + lps.Interface()->SetRatioInfo(&inSize, &outSize); + +#ifdef Z7_USE_ZSTD_ORIG_DECODER + if (hres == S_OK) + opRes = NExtract::NOperationResult::kOK; +#else + if (decoder->ResInfo.decode_SRes == SZ_ERROR_CRC) + { + opRes = NExtract::NOperationResult::kCRCError; + } + else if (decoder->ResInfo.decode_SRes == SZ_ERROR_NO_ARCHIVE) + { + _isArc = false; + opRes = NExtract::NOperationResult::kIsNotArc; + } + else if (decoder->ResInfo.decode_SRes == SZ_ERROR_INPUT_EOF) + opRes = NExtract::NOperationResult::kUnexpectedEnd; + else + { + if (hres == S_OK && decoder->ResInfo.decode_SRes == SZ_OK) + opRes = NExtract::NOperationResult::kOK; + if (decoder->ResInfo.extraSize) + { + // if (inSize == 0) _isArc = false; + opRes = NExtract::NOperationResult::kDataAfterEnd; + } + /* + if (decoder->ResInfo.unexpededEnd) + opRes = NExtract::NOperationResult::kUnexpectedEnd; + */ + } +#endif + } + else if (hres == E_NOTIMPL) + { + opRes = NExtract::NOperationResult::kUnsupportedMethod; + } + else + return hres; + } + + return extractCallback->SetOperationResult(opRes); + + COM_TRY_END +} + + + +Z7_COM7F_IMF(CHandler::SetProperties(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps)) +{ + // return _props.SetProperties(names, values, numProps); + // _smallMode = false; + _disableHash = false; + _parseMode = false; + // _parseMode = true; // for debug +#ifdef Z7_USE_ZSTD_COMPRESSION + _props.Init(); +#endif + + for (UInt32 i = 0; i < numProps; i++) + { + UString name = names[i]; + const PROPVARIANT &value = values[i]; + + if (name.IsEqualTo("parse")) + { + bool parseMode = true; + RINOK(PROPVARIANT_to_bool(value, parseMode)) + _parseMode = parseMode; + continue; + } + if (name.IsPrefixedBy_Ascii_NoCase("crc")) + { + name.Delete(0, 3); + UInt32 crcSize = 4; + RINOK(ParsePropToUInt32(name, value, crcSize)) + if (crcSize == 0) + _disableHash = true; + else if (crcSize == 4) + _disableHash = false; + else + return E_INVALIDARG; + continue; + } +#ifdef Z7_USE_ZSTD_COMPRESSION + /* + if (name.IsEqualTo("small")) + { + bool smallMode = true; + RINOK(PROPVARIANT_to_bool(value, smallMode)) + _smallMode = smallMode; + continue; + } + */ + RINOK(_props.SetProperty(names[i], value)) +#endif + } + return S_OK; +} + + + + +#ifdef Z7_USE_ZSTD_COMPRESSION + +Z7_COM7F_IMF(CHandler::GetFileTimeType(UInt32 *timeType)) +{ + *timeType = GET_FileTimeType_NotDefined_for_GetFileTimeType; + // *timeType = NFileTimeType::kUnix; + return S_OK; +} + + +Z7_COM7F_IMF(CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numItems, + IArchiveUpdateCallback *updateCallback)) +{ + COM_TRY_BEGIN + + if (numItems != 1) + return E_INVALIDARG; + { + CMyComPtr setRestriction; + outStream->QueryInterface(IID_IStreamSetRestriction, (void **)&setRestriction); + if (setRestriction) + RINOK(setRestriction->SetRestriction(0, 0)) + } + Int32 newData, newProps; + UInt32 indexInArchive; + if (!updateCallback) + return E_FAIL; + RINOK(updateCallback->GetUpdateItemInfo(0, &newData, &newProps, &indexInArchive)) + + if (IntToBool(newProps)) + { + { + NCOM::CPropVariant prop; + RINOK(updateCallback->GetProperty(0, kpidIsDir, &prop)) + if (prop.vt != VT_EMPTY) + if (prop.vt != VT_BOOL || prop.boolVal != VARIANT_FALSE) + return E_INVALIDARG; + } + } + + if (IntToBool(newData)) + { + UInt64 size; + { + NCOM::CPropVariant prop; + RINOK(updateCallback->GetProperty(0, kpidSize, &prop)) + if (prop.vt != VT_UI8) + return E_INVALIDARG; + size = prop.uhVal.QuadPart; + } + + if (!_props.MethodName.IsEmpty() + && !_props.MethodName.IsEqualTo_Ascii_NoCase("zstd")) + return E_INVALIDARG; + + { + CMyComPtr fileInStream; + RINOK(updateCallback->GetStream(0, &fileInStream)) + if (!fileInStream) + return S_FALSE; + { + CMyComPtr streamGetSize; + fileInStream.QueryInterface(IID_IStreamGetSize, &streamGetSize); + if (streamGetSize) + { + UInt64 size2; + if (streamGetSize->GetSize(&size2) == S_OK) + size = size2; + } + } + RINOK(updateCallback->SetTotal(size)) + + CMethodProps props2 = _props; + +#ifndef Z7_ST + /* + CSingleMethodProps (_props) + derives from + CMethodProps (props2) + So we transfer additional variable (num Threads) to CMethodProps list of properties + */ + + UInt32 numThreads = _props._numThreads; + + if (numThreads > Z7_ZSTDMT_NBWORKERS_MAX) + numThreads = Z7_ZSTDMT_NBWORKERS_MAX; + + if (_props.FindProp(NCoderPropID::kNumThreads) < 0) + { + if (!_props._numThreads_WasForced + && numThreads >= 1 + && _props._memUsage_WasSet) + { + NCompress::NZstd::CEncoderProps zstdProps; + RINOK(zstdProps.SetFromMethodProps(_props)) + ZstdEncProps_NormalizeFull(&zstdProps.EncProps); + numThreads = ZstdEncProps_GetNumThreads_for_MemUsageLimit( + &zstdProps.EncProps, _props._memUsage_Compress, numThreads); + } + props2.AddProp_NumThreads(numThreads); + } + +#endif // Z7_ST + + CMyComPtr2_Create lps; + lps->Init(updateCallback, true); + { + CMyComPtr2_Create encoder; + // size = 1 << 24; // for debug + RINOK(props2.SetCoderProps(encoder.ClsPtr(), size != (UInt64)(Int64)-1 ? &size : NULL)) + // encoderSpec->_props.SmallFileOpt = _smallMode; + // we must set kExpectedDataSize just before Code(). + encoder->SrcSizeHint64 = size; + /* + CMyComPtr optProps; + _compressEncoder->QueryInterface(IID_ICompressSetCoderPropertiesOpt, (void**)&optProps); + if (optProps) + { + PROPID propID = NCoderPropID::kExpectedDataSize; + NWindows::NCOM::CPropVariant prop = (UInt64)size; + // RINOK(optProps->SetCoderPropertiesOpt(&propID, &prop, 1)) + RINOK(encoderSpec->SetCoderPropertiesOpt(&propID, &prop, 1)) + } + */ + RINOK(encoder.Interface()->Code(fileInStream, outStream, NULL, NULL, lps)) + } + } + return updateCallback->SetOperationResult(NArchive::NUpdate::NOperationResult::kOK); + } + + if (indexInArchive != 0) + return E_INVALIDARG; + + CMyComPtr2_Create lps; + lps->Init(updateCallback, true); + + CMyComPtr opCallback; + updateCallback->QueryInterface(IID_IArchiveUpdateCallbackFile, (void **)&opCallback); + if (opCallback) + { + RINOK(opCallback->ReportOperation( + NEventIndexType::kInArcIndex, 0, + NUpdateNotifyOp::kReplicate)) + } + + if (_stream) + RINOK(_stream->Seek(0, STREAM_SEEK_SET, NULL)) + + return NCompress::CopyStream(_stream, outStream, lps); + + COM_TRY_END +} +#endif + + + +#ifndef Z7_USE_ZSTD_COMPRESSION +#undef IMP_CreateArcOut +#define IMP_CreateArcOut +#undef CreateArcOut +#define CreateArcOut NULL +#endif + +#ifdef Z7_USE_ZSTD_COMPRESSION +REGISTER_ARC_IO( + "zstd2", "zst tzst", "* .tar", 0xe + 1, + k_Signature, 0 + , NArcInfoFlags::kKeepName + , 0 + , NULL) +#else +REGISTER_ARC_IO( + "zstd", "zst tzst", "* .tar", 0xe, + k_Signature, 0 + , NArcInfoFlags::kKeepName + , 0 + , NULL) +#endif + +}} diff --git a/CPP/7zip/Asm.mak b/CPP/7zip/Asm.mak index c4073e8..4ee0823 100644 --- a/CPP/7zip/Asm.mak +++ b/CPP/7zip/Asm.mak @@ -1,8 +1,11 @@ !IFDEF ASM_OBJS -!IF "$(CPU)" == "ARM" -$(ASM_OBJS): ../../../../Asm/Arm/$(*B).asm +!IF "$(PLATFORM)" == "arm64" +$(ASM_OBJS): ../../../../Asm/arm64/$(*B).S + $(COMPL_ASM_CLANG) +!ELSEIF "$(PLATFORM)" == "arm" +$(ASM_OBJS): ../../../../Asm/arm/$(*B).asm $(COMPL_ASM) -!ELSEIF "$(CPU)" != "IA64" && "$(CPU)" != "MIPS" && "$(CPU)" != "ARM64" +!ELSEIF "$(PLATFORM)" != "ia64" && "$(PLATFORM)" != "mips" $(ASM_OBJS): ../../../../Asm/x86/$(*B).asm $(COMPL_ASM) !ENDIF diff --git a/CPP/7zip/Bundles/Alone/Alone.dsp b/CPP/7zip/Bundles/Alone/Alone.dsp index 65c81c4..beed5a7 100644 --- a/CPP/7zip/Bundles/Alone/Alone.dsp +++ b/CPP/7zip/Bundles/Alone/Alone.dsp @@ -69,8 +69,7 @@ LINK32=link.exe # PROP Ignore_Export_Lib 0 # PROP Target_Dir "" # ADD BASE CPP /nologo /W3 /Gm /GX /ZI /Od /D "WIN32" /D "_DEBUG" /D "_CONSOLE" /D "_MBCS" /YX /FD /GZ /c -# ADD CPP /nologo /Gz /MDd /W4 /Gm /GX /ZI /Od /I "..\..\..\\" /D "_DEBUG" /D "_MBCS" /D "WIN32" /D "_CONSOLE" /D "Z7_LONG_PATH" /D "Z7_LARGE_PAGES" /D "Z7_DEVICE_FILE" /Yu"StdAfx.h" /FD /GZ /c -# SUBTRACT CPP /WX +# ADD CPP /nologo /Gz /MDd /W4 /WX /Gm /GX /ZI /Od /I "..\..\..\\" /D "_DEBUG" /D "_MBCS" /D "WIN32" /D "_CONSOLE" /D "Z7_LONG_PATH" /D "Z7_LARGE_PAGES" /D "Z7_DEVICE_FILE" /Yu"StdAfx.h" /FD /GZ /c # ADD BASE RSC /l 0x419 /d "_DEBUG" # ADD RSC /l 0x419 /d "_DEBUG" BSC32=bscmake.exe @@ -194,6 +193,7 @@ SOURCE=..\..\UI\Console\List.h # Begin Source File SOURCE=..\..\UI\Console\Main.cpp +# ADD CPP /D "Z7_PROG_VARIANT_A" # End Source File # Begin Source File @@ -274,6 +274,10 @@ SOURCE=..\..\..\Common\Common.h # End Source File # Begin Source File +SOURCE=..\..\..\Common\Common0.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Common\ComTry.h # End Source File # Begin Source File @@ -450,6 +454,10 @@ SOURCE=..\..\..\Common\Wildcard.h # End Source File # Begin Source File +SOURCE=..\..\..\Common\Xxh64Reg.cpp +# End Source File +# Begin Source File + SOURCE=..\..\..\Common\XzCrc64Init.cpp # End Source File # Begin Source File @@ -1190,6 +1198,20 @@ SOURCE=..\..\Compress\Lzx.h # Begin Source File SOURCE=..\..\Compress\LzxDecoder.cpp + +!IF "$(CFG)" == "Alone - Win32 Release" + +# ADD CPP /O2 +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 Debug" + +!ELSEIF "$(CFG)" == "Alone - Win32 ReleaseU" + +!ELSEIF "$(CFG)" == "Alone - Win32 DebugU" + +!ENDIF + # End Source File # Begin Source File @@ -1202,6 +1224,20 @@ SOURCE=..\..\Compress\LzxDecoder.h # Begin Source File SOURCE=..\..\Compress\QuantumDecoder.cpp + +!IF "$(CFG)" == "Alone - Win32 Release" + +# ADD CPP /O2 +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 Debug" + +!ELSEIF "$(CFG)" == "Alone - Win32 ReleaseU" + +!ELSEIF "$(CFG)" == "Alone - Win32 DebugU" + +!ENDIF + # End Source File # Begin Source File @@ -1300,6 +1336,14 @@ SOURCE=..\..\Compress\XzEncoder.cpp SOURCE=..\..\Compress\XzEncoder.h # End Source File +# Begin Source File + +SOURCE=..\..\Compress\ZstdDecoder.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\Compress\ZstdDecoder.h +# End Source File # End Group # Begin Group "Archive" @@ -1486,6 +1530,8 @@ SOURCE=..\..\Archive\Zip\ZipCompressionMode.h # Begin Source File SOURCE=..\..\Archive\Zip\ZipHandler.cpp +# ADD CPP /D "Z7_ZIP_LZFSE_DISABLE" +# SUBTRACT CPP /YX /Yc /Yu # End Source File # Begin Source File @@ -1692,6 +1738,10 @@ SOURCE=..\..\Archive\SplitHandler.cpp SOURCE=..\..\Archive\XzHandler.cpp # End Source File +# Begin Source File + +SOURCE=..\..\Archive\ZstdHandler.cpp +# End Source File # End Group # Begin Group "UI Common" @@ -3077,6 +3127,10 @@ SOURCE=..\..\..\..\C\Ppmd8Enc.c # End Source File # Begin Source File +SOURCE=..\..\..\..\C\Precomp.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\RotateDefs.h # End Source File # Begin Source File @@ -3229,6 +3283,62 @@ SOURCE=..\..\..\..\C\Threads.c SOURCE=..\..\..\..\C\Threads.h # End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\Xxh64.c + +!IF "$(CFG)" == "Alone - Win32 Release" + +# ADD CPP /O2 +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 Debug" + +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 ReleaseU" + +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 DebugU" + +# SUBTRACT CPP /YX /Yc /Yu + +!ENDIF + +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\Xxh64.h +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\ZstdDec.c + +!IF "$(CFG)" == "Alone - Win32 Release" + +# ADD CPP /O2 +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 Debug" + +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 ReleaseU" + +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 DebugU" + +# SUBTRACT CPP /YX /Yc /Yu + +!ENDIF + +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\ZstdDec.h +# End Source File # End Group # End Target # End Project diff --git a/CPP/7zip/Bundles/Alone/makefile b/CPP/7zip/Bundles/Alone/makefile index bad953a..7547590 100644 --- a/CPP/7zip/Bundles/Alone/makefile +++ b/CPP/7zip/Bundles/Alone/makefile @@ -1,4 +1,10 @@ PROG = 7za.exe + +CFLAGS = $(CFLAGS) -DZ7_ZIP_LZFSE_DISABLE +# -DZ7_PROG_VARIANT_A +# CONSOLE_VARIANT_FLAGS=-DZ7_PROG_VARIANT_A +# ZIP_FLAGS=-DZ7_ZIP_LZFSE_DISABLE + # USE_C_AES = 1 # USE_C_SHA = 1 # USE_C_LZFINDOPT = 1 @@ -20,10 +26,11 @@ COMMON_OBJS = \ $O\UTFConvert.obj \ $O\MyVector.obj \ $O\Wildcard.obj \ - $O\XzCrc64Init.obj \ - $O\XzCrc64Reg.obj \ $O\Sha1Reg.obj \ $O\Sha256Reg.obj \ + $O\Xxh64Reg.obj \ + $O\XzCrc64Init.obj \ + $O\XzCrc64Reg.obj \ WIN_OBJS = \ $O\DLL.obj \ @@ -76,6 +83,7 @@ AR_OBJS = \ $O\LzmaHandler.obj \ $O\SplitHandler.obj \ $O\XzHandler.obj \ + $O\ZstdHandler.obj \ AR_COMMON_OBJS = \ $O\CoderMixer2.obj \ @@ -169,6 +177,10 @@ COMPRESS_OBJS = \ $O\ShrinkDecoder.obj \ $O\XzDecoder.obj \ $O\XzEncoder.obj \ + $O\ZstdDecoder.obj \ + +# $O\LzfseDecoder.obj \ +# $O\ZstdRegister.obj \ CRYPTO_OBJS = \ $O\7zAes.obj \ @@ -212,10 +224,12 @@ C_OBJS = \ $O\Sort.obj \ $O\SwapBytes.obj \ $O\Threads.obj \ + $O\Xxh64.obj \ $O\Xz.obj \ $O\XzDec.obj \ $O\XzEnc.obj \ $O\XzIn.obj \ + $O\ZstdDec.obj \ !include "../../UI/Console/Console.mak" diff --git a/CPP/7zip/Bundles/Alone/makefile.gcc b/CPP/7zip/Bundles/Alone/makefile.gcc index cb7e828..621c0ce 100644 --- a/CPP/7zip/Bundles/Alone/makefile.gcc +++ b/CPP/7zip/Bundles/Alone/makefile.gcc @@ -1,18 +1,13 @@ PROG = 7za +CONSOLE_VARIANT_FLAGS=-DZ7_PROG_VARIANT_A +ZIP_FLAGS=-DZ7_ZIP_LZFSE_DISABLE # IS_X64 = 1 # USE_ASM = 1 # ST_MODE = 1 -include ../../LzmaDec_gcc.mak - - -LOCAL_FLAGS_ST = -MT_OBJS = - - ifdef SystemDrive IS_MINGW = 1 else @@ -22,6 +17,13 @@ IS_MINGW = 1 endif endif +include ../../LzmaDec_gcc.mak + + +LOCAL_FLAGS_ST = +MT_OBJS = + + ifdef ST_MODE LOCAL_FLAGS_ST = -DZ7_ST @@ -36,13 +38,14 @@ else MT_OBJS = \ $O/LzFindMt.o \ - $O/StreamBinder.o \ - $O/Synchronization.o \ - $O/VirtThread.o \ + $O/LzFindOpt.o \ + $O/Threads.o \ $O/MemBlocks.o \ $O/OutMemStream.o \ $O/ProgressMt.o \ - $O/Threads.o \ + $O/StreamBinder.o \ + $O/Synchronization.o \ + $O/VirtThread.o \ endif @@ -53,10 +56,11 @@ LOCAL_FLAGS_SYS = ifdef IS_MINGW LOCAL_FLAGS_SYS = \ - -DZ7_LARGE_PAGES \ - -DZ7_LONG_PATH \ -DZ7_DEVICE_FILE \ +# -DZ7_LARGE_PAGES \ +# -DZ7_LONG_PATH \ + SYS_OBJS = \ $O/FileSystem.o \ $O/Registry.o \ @@ -77,7 +81,6 @@ LOCAL_FLAGS = \ $(LOCAL_FLAGS_SYS) \ - CONSOLE_OBJS = \ $O/BenchCon.o \ $O/ConsoleClose.o \ @@ -122,22 +125,22 @@ COMMON_OBJS = \ $O/ListFileUtils.o \ $O/LzFindPrepare.o \ $O/MyString.o \ + $O/MyVector.o \ $O/NewHandler.o \ - $O/StdInStream.o \ - $O/StdOutStream.o \ $O/Sha1Prepare.o \ $O/Sha1Reg.o \ $O/Sha256Prepare.o \ $O/Sha256Reg.o \ + $O/StdInStream.o \ + $O/StdOutStream.o \ $O/StringConvert.o \ $O/StringToInt.o \ $O/UTFConvert.o \ - $O/MyVector.o \ $O/Wildcard.o \ + $O/Xxh64Reg.o \ $O/XzCrc64Init.o \ $O/XzCrc64Reg.o \ - WIN_OBJS = \ $O/ErrorMsg.o \ $O/FileDir.o \ @@ -152,7 +155,6 @@ WIN_OBJS = \ $O/SystemInfo.o \ $O/TimeUtils.o \ - 7ZIP_COMMON_OBJS = \ $O/CreateCoder.o \ $O/CWrappers.o \ @@ -179,6 +181,7 @@ AR_OBJS = \ $O/LzmaHandler.o \ $O/SplitHandler.o \ $O/XzHandler.o \ + $O/ZstdHandler.o \ AR_COMMON_OBJS = \ $O/CoderMixer2.o \ @@ -269,6 +272,10 @@ COMPRESS_OBJS = \ $O/ShrinkDecoder.o \ $O/XzDecoder.o \ $O/XzEncoder.o \ + $O/ZstdDecoder.o \ + +# $O/LzfseDecoder.o \ +# $O/ZstdRegister.o CRYPTO_OBJS = \ $O/7zAes.o \ @@ -283,7 +290,11 @@ CRYPTO_OBJS = \ $O/ZipStrong.o \ C_OBJS = \ + $O/7zCrc.o \ + $O/7zCrcOpt.o \ $O/7zStream.o \ + $O/Aes.o \ + $O/AesOpt.o \ $O/Alloc.o \ $O/Bcj2.o \ $O/Bcj2Enc.o \ @@ -295,7 +306,6 @@ C_OBJS = \ $O/Delta.o \ $O/HuffEnc.o \ $O/LzFind.o \ - $O/LzFindOpt.o \ $O/Lzma2Dec.o \ $O/Lzma2DecMt.o \ $O/Lzma2Enc.o \ @@ -309,22 +319,21 @@ C_OBJS = \ $O/Ppmd8.o \ $O/Ppmd8Dec.o \ $O/Ppmd8Enc.o \ + $O/Sha1.o \ + $O/Sha1Opt.o \ + $O/Sha256.o \ + $O/Sha256Opt.o \ $O/Sort.o \ $O/SwapBytes.o \ + $O/Xxh64.o \ $O/Xz.o \ $O/XzDec.o \ $O/XzEnc.o \ $O/XzIn.o \ $O/XzCrc64.o \ $O/XzCrc64Opt.o \ - $O/7zCrc.o \ - $O/7zCrcOpt.o \ - $O/Aes.o \ - $O/AesOpt.o \ - $O/Sha256.o \ - $O/Sha256Opt.o \ - $O/Sha1.o \ - $O/Sha1Opt.o \ + $O/ZstdDec.o \ + OBJS = \ $(LZMA_DEC_OPT_OBJS) \ diff --git a/CPP/7zip/Bundles/Alone2/makefile b/CPP/7zip/Bundles/Alone2/makefile index 4cb7718..af89d66 100644 --- a/CPP/7zip/Bundles/Alone2/makefile +++ b/CPP/7zip/Bundles/Alone2/makefile @@ -1,7 +1,9 @@ PROG = 7zz.exe # USE_C_AES = 1 # USE_C_SHA = 1 + CFLAGS = $(CFLAGS) -DZ7_PROG_VARIANT_Z +# CONSOLE_VARIANT_FLAGS=-DZ7_PROG_VARIANT_Z !include "../Format7zF/Arc.mak" !include "../../UI/Console/Console.mak" diff --git a/CPP/7zip/Bundles/Alone2/makefile.gcc b/CPP/7zip/Bundles/Alone2/makefile.gcc index f767b0d..2785aea 100644 --- a/CPP/7zip/Bundles/Alone2/makefile.gcc +++ b/CPP/7zip/Bundles/Alone2/makefile.gcc @@ -1,11 +1,11 @@ PROG = 7zz +CONSOLE_VARIANT_FLAGS=-DZ7_PROG_VARIANT_Z + # IS_X64 = 1 # USE_ASM = 1 # ST_MODE = 1 -CONSOLE_VARIANT_FLAGS=-DZ7_PROG_VARIANT_Z - include ../Format7zF/Arc_gcc.mak ifdef SystemDrive @@ -17,13 +17,16 @@ IS_MINGW = 1 endif endif +LOCAL_FLAGS_SYS = + ifdef IS_MINGW LOCAL_FLAGS_SYS = \ - -DZ7_LARGE_PAGES \ - -DZ7_LONG_PATH \ -DZ7_DEVICE_FILE \ +# -DZ7_LONG_PATH \ +# -DZ7_DEVICE_FILE \ + SYS_OBJS = \ $O/FileSystem.o \ $O/Registry.o \ @@ -67,7 +70,6 @@ UI_COMMON_OBJS = \ $O/UpdatePair.o \ $O/UpdateProduce.o \ - CONSOLE_OBJS = \ $O/BenchCon.o \ $O/ConsoleClose.o \ diff --git a/CPP/7zip/Bundles/Alone7z/Alone.dsp b/CPP/7zip/Bundles/Alone7z/Alone.dsp index ef4ec60..317a88f 100644 --- a/CPP/7zip/Bundles/Alone7z/Alone.dsp +++ b/CPP/7zip/Bundles/Alone7z/Alone.dsp @@ -270,6 +270,10 @@ SOURCE=..\..\..\Common\Common.h # End Source File # Begin Source File +SOURCE=..\..\..\Common\Common0.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Common\ComTry.h # End Source File # Begin Source File @@ -342,6 +346,10 @@ SOURCE=..\..\..\Common\MyInitGuid.h # End Source File # Begin Source File +SOURCE=..\..\..\Common\MyLinux.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Common\MyString.cpp # End Source File # Begin Source File @@ -366,6 +374,10 @@ SOURCE=..\..\..\Common\MyVector.h # End Source File # Begin Source File +SOURCE=..\..\MyVersion.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Common\MyWindows.cpp # End Source File # Begin Source File @@ -566,6 +578,10 @@ SOURCE=..\..\..\Windows\Registry.h # End Source File # Begin Source File +SOURCE=..\..\..\Windows\SecurityUtils.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Windows\Synchronization.h # End Source File # Begin Source File @@ -1130,6 +1146,10 @@ SOURCE=..\..\UI\Common\DefaultName.h # End Source File # Begin Source File +SOURCE=..\..\UI\Common\DirItem.h +# End Source File +# Begin Source File + SOURCE=..\..\UI\Common\EnumDirItems.cpp # End Source File # Begin Source File @@ -1493,6 +1513,10 @@ SOURCE=..\..\..\..\C\7zTypes.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\7zVersion.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\7zWindows.h # End Source File # Begin Source File @@ -1611,7 +1635,26 @@ SOURCE=..\..\..\..\C\Bcj2Enc.c # Begin Source File SOURCE=..\..\..\..\C\Bra.c + +!IF "$(CFG)" == "Alone - Win32 Release" + +# ADD CPP /O2 +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 Debug" + +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 ReleaseU" + # SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 DebugU" + +# SUBTRACT CPP /YX /Yc /Yu + +!ENDIF + # End Source File # Begin Source File @@ -1930,6 +1973,10 @@ SOURCE=..\..\..\..\C\MtDec.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\Precomp.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\RotateDefs.h # End Source File # Begin Source File diff --git a/CPP/7zip/Bundles/Alone7z/makefile b/CPP/7zip/Bundles/Alone7z/makefile index 1f857c5..89584e1 100644 --- a/CPP/7zip/Bundles/Alone7z/makefile +++ b/CPP/7zip/Bundles/Alone7z/makefile @@ -2,8 +2,13 @@ PROG = 7zr.exe # USE_C_AES = 1 # USE_C_SHA = 1 +# USE_C_CRC64 = 1 +# USE_C_CRC = 1 +# NO_ASM_GNU=1 +# NO_ASM=1 CFLAGS = $(CFLAGS) -DZ7_PROG_VARIANT_R +# CONSOLE_VARIANT_FLAGS=-DZ7_PROG_VARIANT_R COMMON_OBJS = \ $O\CommandLineParser.obj \ diff --git a/CPP/7zip/Bundles/Alone7z/makefile.gcc b/CPP/7zip/Bundles/Alone7z/makefile.gcc index 6d92c19..a31475e 100644 --- a/CPP/7zip/Bundles/Alone7z/makefile.gcc +++ b/CPP/7zip/Bundles/Alone7z/makefile.gcc @@ -37,9 +37,11 @@ else MT_OBJS = \ $O/LzFindMt.o \ $O/LzFindOpt.o \ + $O/Threads.o \ $O/StreamBinder.o \ $O/VirtThread.o \ - $O/Threads.o \ + + @@ -52,10 +54,11 @@ LOCAL_FLAGS_SYS = ifdef IS_MINGW LOCAL_FLAGS_SYS = \ - -DZ7_LARGE_PAGES \ - -DZ7_LONG_PATH \ -DZ7_DEVICE_FILE \ +# -DZ7_LARGE_PAGES \ +# -DZ7_LONG_PATH \ + SYS_OBJS = \ $O/FileSystem.o \ $O/Registry.o \ @@ -79,7 +82,6 @@ LOCAL_FLAGS = \ CONSOLE_OBJS = \ $O/BenchCon.o \ $O/ConsoleClose.o \ - $O/DynLimBuf.o \ $O/ExtractCallbackConsole.o \ $O/HashCon.o \ $O/List.o \ @@ -116,6 +118,7 @@ COMMON_OBJS = \ $O/CommandLineParser.o \ $O/CRC.o \ $O/CrcReg.o \ + $O/DynLimBuf.o \ $O/IntToString.o \ $O/ListFileUtils.o \ $O/LzFindPrepare.o \ @@ -124,10 +127,10 @@ COMMON_OBJS = \ $O/NewHandler.o \ $O/Sha256Prepare.o \ $O/Sha256Reg.o \ - $O/StringConvert.o \ - $O/StringToInt.o \ $O/StdInStream.o \ $O/StdOutStream.o \ + $O/StringConvert.o \ + $O/StringToInt.o \ $O/UTFConvert.o \ $O/Wildcard.o \ $O/XzCrc64Init.o \ @@ -225,7 +228,11 @@ CRYPTO_OBJS = \ $O/RandGen.o \ C_OBJS = \ + $O/7zCrc.o \ + $O/7zCrcOpt.o \ $O/7zStream.o \ + $O/Aes.o \ + $O/AesOpt.o \ $O/Alloc.o \ $O/Bcj2.o \ $O/Bcj2Enc.o \ @@ -251,10 +258,6 @@ C_OBJS = \ $O/XzIn.o \ $O/XzCrc64.o \ $O/XzCrc64Opt.o \ - $O/7zCrc.o \ - $O/7zCrcOpt.o \ - $O/Aes.o \ - $O/AesOpt.o \ OBJS = \ diff --git a/CPP/7zip/Bundles/Fm/FM.dsp b/CPP/7zip/Bundles/Fm/FM.dsp index 402dd2c..6e55d92 100644 --- a/CPP/7zip/Bundles/Fm/FM.dsp +++ b/CPP/7zip/Bundles/Fm/FM.dsp @@ -661,6 +661,18 @@ SOURCE=..\..\UI\FileManager\BrowseDialog.h # End Source File # Begin Source File +SOURCE=..\..\UI\FileManager\BrowseDialog2.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\UI\FileManager\BrowseDialog2.h +# End Source File +# Begin Source File + +SOURCE=..\..\UI\FileManager\BrowseDialog2Res.h +# End Source File +# Begin Source File + SOURCE=..\..\UI\FileManager\ComboDialog.cpp # End Source File # Begin Source File @@ -705,6 +717,18 @@ SOURCE=..\..\UI\FileManager\ListViewDialog.h # End Source File # Begin Source File +SOURCE=..\..\UI\FileManager\MemDialog.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\UI\FileManager\MemDialog.h +# End Source File +# Begin Source File + +SOURCE=..\..\UI\FileManager\MemDialogRes.h +# End Source File +# Begin Source File + SOURCE=..\..\UI\FileManager\MessagesDialog.cpp # End Source File # Begin Source File diff --git a/CPP/7zip/Bundles/Fm/makefile b/CPP/7zip/Bundles/Fm/makefile index 33b5320..de74f76 100644 --- a/CPP/7zip/Bundles/Fm/makefile +++ b/CPP/7zip/Bundles/Fm/makefile @@ -1,6 +1,6 @@ PROG = 7zFM.exe -CFLAGS = $(CFLAGS) -DZ7_LARGE_PAGES +# CFLAGS = $(CFLAGS) -DZ7_LARGE_PAGES !include "../Format7zF/Arc.mak" diff --git a/CPP/7zip/Bundles/Format7z/makefile b/CPP/7zip/Bundles/Format7z/makefile index d20a3c1..fe6f94d 100644 --- a/CPP/7zip/Bundles/Format7z/makefile +++ b/CPP/7zip/Bundles/Format7z/makefile @@ -25,6 +25,7 @@ WIN_OBJS = \ $O\PropVariant.obj \ $O\Synchronization.obj \ $O\System.obj \ + $O\TimeUtils.obj \ 7ZIP_COMMON_OBJS = \ $O\CreateCoder.obj \ diff --git a/CPP/7zip/Bundles/Format7zF/Arc.mak b/CPP/7zip/Bundles/Format7zF/Arc.mak index 37b400e..3d8a430 100644 --- a/CPP/7zip/Bundles/Format7zF/Arc.mak +++ b/CPP/7zip/Bundles/Format7zF/Arc.mak @@ -15,6 +15,7 @@ COMMON_OBJS = \ $O\StringToInt.obj \ $O\UTFConvert.obj \ $O\Wildcard.obj \ + $O\Xxh64Reg.obj \ $O\XzCrc64Init.obj \ $O\XzCrc64Reg.obj \ @@ -98,6 +99,7 @@ AR_OBJS = \ $O\XarHandler.obj \ $O\XzHandler.obj \ $O\ZHandler.obj \ + $O\ZstdHandler.obj \ AR_COMMON_OBJS = \ $O\CoderMixer2.obj \ @@ -233,6 +235,7 @@ COMPRESS_OBJS = \ $O\ZlibDecoder.obj \ $O\ZlibEncoder.obj \ $O\ZDecoder.obj \ + $O\ZstdDecoder.obj \ CRYPTO_OBJS = \ $O\7zAes.obj \ @@ -283,10 +286,12 @@ C_OBJS = \ $O\Sort.obj \ $O\SwapBytes.obj \ $O\Threads.obj \ + $O\Xxh64.obj \ $O\Xz.obj \ $O\XzDec.obj \ $O\XzEnc.obj \ $O\XzIn.obj \ + $O\ZstdDec.obj \ !include "../../Aes.mak" !include "../../Crc.mak" diff --git a/CPP/7zip/Bundles/Format7zF/Arc_gcc.mak b/CPP/7zip/Bundles/Format7zF/Arc_gcc.mak index 404d917..ff5a3f9 100644 --- a/CPP/7zip/Bundles/Format7zF/Arc_gcc.mak +++ b/CPP/7zip/Bundles/Format7zF/Arc_gcc.mak @@ -27,13 +27,13 @@ else MT_OBJS = \ $O/LzFindMt.o \ $O/LzFindOpt.o \ - $O/StreamBinder.o \ - $O/Synchronization.o \ - $O/VirtThread.o \ + $O/Threads.o \ $O/MemBlocks.o \ $O/OutMemStream.o \ $O/ProgressMt.o \ - $O/Threads.o \ + $O/StreamBinder.o \ + $O/Synchronization.o \ + $O/VirtThread.o \ endif @@ -58,6 +58,7 @@ COMMON_OBJS = \ $O/StringToInt.o \ $O/UTFConvert.o \ $O/Wildcard.o \ + $O/Xxh64Reg.o \ $O/XzCrc64Init.o \ $O/XzCrc64Reg.o \ @@ -135,6 +136,10 @@ AR_OBJS = \ $O/XarHandler.o \ $O/XzHandler.o \ $O/ZHandler.o \ + $O/ZstdHandler.o \ + +# $O/AvbHandler.o +# $O/LvmHandler.o AR_COMMON_OBJS = \ $O/CoderMixer2.o \ @@ -269,6 +274,7 @@ COMPRESS_OBJS = \ $O/ZlibDecoder.o \ $O/ZlibEncoder.o \ $O/ZDecoder.o \ + $O/ZstdDecoder.o \ ifdef DISABLE_RAR DISABLE_RAR_COMPRESS=1 @@ -309,7 +315,11 @@ endif C_OBJS = \ $O/7zBuf2.o \ + $O/7zCrc.o \ + $O/7zCrcOpt.o \ $O/7zStream.o \ + $O/Aes.o \ + $O/AesOpt.o \ $O/Alloc.o \ $O/Bcj2.o \ $O/Bcj2Enc.o \ @@ -336,22 +346,20 @@ C_OBJS = \ $O/Ppmd8.o \ $O/Ppmd8Dec.o \ $O/Ppmd8Enc.o \ + $O/Sha1.o \ + $O/Sha1Opt.o \ + $O/Sha256.o \ + $O/Sha256Opt.o \ $O/Sort.o \ + $O/SwapBytes.o \ + $O/Xxh64.o \ $O/Xz.o \ $O/XzDec.o \ $O/XzEnc.o \ $O/XzIn.o \ $O/XzCrc64.o \ $O/XzCrc64Opt.o \ - $O/7zCrc.o \ - $O/7zCrcOpt.o \ - $O/Aes.o \ - $O/AesOpt.o \ - $O/Sha256.o \ - $O/Sha256Opt.o \ - $O/Sha1.o \ - $O/Sha1Opt.o \ - $O/SwapBytes.o \ + $O/ZstdDec.o \ ARC_OBJS = \ $(LZMA_DEC_OPT_OBJS) \ diff --git a/CPP/7zip/Bundles/Format7zF/Format7z.dsp b/CPP/7zip/Bundles/Format7zF/Format7z.dsp index a0c5d83..6e28288 100644 --- a/CPP/7zip/Bundles/Format7zF/Format7z.dsp +++ b/CPP/7zip/Bundles/Format7zF/Format7z.dsp @@ -195,6 +195,10 @@ SOURCE=..\..\Archive\Icons\z.ico SOURCE=..\..\Archive\Icons\zip.ico # End Source File +# Begin Source File + +SOURCE=..\..\Archive\Icons\zst.ico +# End Source File # End Group # Begin Source File @@ -231,10 +235,18 @@ SOURCE=.\StdAfx.h # PROP Default_Filter "" # Begin Source File +SOURCE=..\..\..\Common\AutoPtr.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Common\Common.h # End Source File # Begin Source File +SOURCE=..\..\..\Common\Common0.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Common\ComTry.h # End Source File # Begin Source File @@ -403,6 +415,10 @@ SOURCE=..\..\..\Common\Wildcard.h # End Source File # Begin Source File +SOURCE=..\..\..\Common\Xxh64Reg.cpp +# End Source File +# Begin Source File + SOURCE=..\..\..\Common\XzCrc64Init.cpp # End Source File # Begin Source File @@ -1069,6 +1085,14 @@ SOURCE=..\..\Compress\ZDecoder.cpp SOURCE=..\..\Compress\ZDecoder.h # End Source File +# Begin Source File + +SOURCE=..\..\Compress\ZstdDecoder.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\Compress\ZstdDecoder.h +# End Source File # End Group # Begin Group "Crypto" @@ -2253,6 +2277,46 @@ SOURCE=..\..\..\..\C\Threads.c SOURCE=..\..\..\..\C\Threads.h # End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\Xxh64.c + +!IF "$(CFG)" == "7z - Win32 Release" + +# ADD CPP /O2 +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "7z - Win32 Debug" + +# SUBTRACT CPP /YX /Yc /Yu + +!ENDIF + +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\Xxh64.h +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\ZstdDec.c + +!IF "$(CFG)" == "7z - Win32 Release" + +# ADD CPP /O2 +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "7z - Win32 Debug" + +# SUBTRACT CPP /YX /Yc /Yu + +!ENDIF + +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\ZstdDec.h +# End Source File # End Group # Begin Group "Archive" @@ -2995,6 +3059,10 @@ SOURCE=..\..\Archive\XzHandler.h SOURCE=..\..\Archive\ZHandler.cpp # End Source File +# Begin Source File + +SOURCE=..\..\Archive\ZstdHandler.cpp +# End Source File # End Group # Begin Group "7zip" diff --git a/CPP/7zip/Bundles/Format7zF/makefile b/CPP/7zip/Bundles/Format7zF/makefile index 3487596..521c8cf 100644 --- a/CPP/7zip/Bundles/Format7zF/makefile +++ b/CPP/7zip/Bundles/Format7zF/makefile @@ -5,7 +5,7 @@ CFLAGS = $(CFLAGS) \ -DZ7_EXTERNAL_CODECS \ !IFNDEF UNDER_CE -CFLAGS = $(CFLAGS) -DZ7_LARGE_PAGES +# CFLAGS = $(CFLAGS) -DZ7_LARGE_PAGES !ENDIF !include "Arc.mak" diff --git a/CPP/7zip/Bundles/Format7zF/makefile.gcc b/CPP/7zip/Bundles/Format7zF/makefile.gcc index fc65795..cc23d0b 100644 --- a/CPP/7zip/Bundles/Format7zF/makefile.gcc +++ b/CPP/7zip/Bundles/Format7zF/makefile.gcc @@ -16,13 +16,15 @@ IS_MINGW = 1 endif endif +LOCAL_FLAGS_SYS = ifdef IS_MINGW -LOCAL_FLAGS_WIN = \ - -DZ7_LARGE_PAGES \ +LOCAL_FLAGS_SYS = \ $(LOCAL_FLAGS_ST) \ +# -DZ7_LARGE_PAGES \ + SYS_OBJS = \ $O/resource.o \ @@ -35,7 +37,7 @@ endif LOCAL_FLAGS = \ -DZ7_EXTERNAL_CODECS \ - $(LOCAL_FLAGS_WIN) \ + $(LOCAL_FLAGS_SYS) \ $(LOCAL_FLAGS_ST) \ diff --git a/CPP/7zip/Bundles/Format7zF/resource.rc b/CPP/7zip/Bundles/Format7zF/resource.rc index 9c797c1..f2950c0 100644 --- a/CPP/7zip/Bundles/Format7zF/resource.rc +++ b/CPP/7zip/Bundles/Format7zF/resource.rc @@ -29,10 +29,11 @@ MY_VERSION_INFO_DLL("7z Plugin" , "7z") 23 ICON "../../Archive/Icons/xz.ico" 24 ICON "../../Archive/Icons/squashfs.ico" 25 ICON "../../Archive/Icons/apfs.ico" +26 ICON "../../Archive/Icons/zst.ico" STRINGTABLE BEGIN - 100 "7z:0 zip:1 rar:3 001:9 cab:7 iso:8 xz:23 txz:23 lzma:16 tar:13 cpio:12 bz2:2 bzip2:2 tbz2:2 tbz:2 gz:14 gzip:14 tgz:14 tpz:14 z:5 taz:5 lzh:6 lha:6 rpm:10 deb:11 arj:4 vhd:20 vhdx:20 wim:15 swm:15 esd:15 fat:21 ntfs:22 dmg:17 hfs:18 xar:19 squashfs:24 apfs:25" + 100 "7z:0 zip:1 rar:3 001:9 cab:7 iso:8 xz:23 txz:23 lzma:16 tar:13 cpio:12 bz2:2 bzip2:2 tbz2:2 tbz:2 gz:14 gzip:14 tgz:14 tpz:14 zst:26 tzst:26 z:5 taz:5 lzh:6 lha:6 rpm:10 deb:11 arj:4 vhd:20 vhdx:20 wim:15 swm:15 esd:15 fat:21 ntfs:22 dmg:17 hfs:18 xar:19 squashfs:24 apfs:25" END diff --git a/CPP/7zip/Bundles/Format7zR/makefile b/CPP/7zip/Bundles/Format7zR/makefile index 2449755..326d043 100644 --- a/CPP/7zip/Bundles/Format7zR/makefile +++ b/CPP/7zip/Bundles/Format7zR/makefile @@ -23,6 +23,7 @@ WIN_OBJS = \ $O\PropVariant.obj \ $O\Synchronization.obj \ $O\System.obj \ + $O\TimeUtils.obj \ 7ZIP_COMMON_OBJS = \ $O\CreateCoder.obj \ diff --git a/CPP/7zip/Bundles/LzmaCon/LzmaAlone.cpp b/CPP/7zip/Bundles/LzmaCon/LzmaAlone.cpp index ac1334f..b4bd2de 100644 --- a/CPP/7zip/Bundles/LzmaCon/LzmaAlone.cpp +++ b/CPP/7zip/Bundles/LzmaCon/LzmaAlone.cpp @@ -501,7 +501,7 @@ static int main2(int numArgs, const char *args[]) const UString &outputName = params[paramIndex++]; outStreamSpec = new COutFileStream; outStream = outStreamSpec; - if (!outStreamSpec->Create(us2fs(outputName), true)) + if (!outStreamSpec->Create_ALWAYS(us2fs(outputName))) { PrintError2("Cannot open output file", outputName); return 1; diff --git a/CPP/7zip/Bundles/LzmaCon/LzmaCon.dsp b/CPP/7zip/Bundles/LzmaCon/LzmaCon.dsp index 5a8911b..258bb6d 100644 --- a/CPP/7zip/Bundles/LzmaCon/LzmaCon.dsp +++ b/CPP/7zip/Bundles/LzmaCon/LzmaCon.dsp @@ -42,7 +42,7 @@ RSC=rc.exe # PROP Ignore_Export_Lib 0 # PROP Target_Dir "" # ADD BASE CPP /nologo /W3 /GX /O2 /D "WIN32" /D "NDEBUG" /D "_CONSOLE" /D "_MBCS" /YX /FD /c -# ADD CPP /nologo /Gr /MD /W4 /WX /GX /O2 /D "WIN32" /D "NDEBUG" /D "_CONSOLE" /D "_MBCS" /Yu"StdAfx.h" /FD /c +# ADD CPP /nologo /Gr /MD /W4 /WX /GX /O2 /D "NDEBUG" /D "WIN32" /D "_CONSOLE" /D "_MBCS" /D "Z7_NO_LONG_PATH" /Yu"StdAfx.h" /FD /c # ADD BASE RSC /l 0x419 /d "NDEBUG" # ADD RSC /l 0x419 /d "NDEBUG" BSC32=bscmake.exe @@ -66,7 +66,7 @@ LINK32=link.exe # PROP Ignore_Export_Lib 0 # PROP Target_Dir "" # ADD BASE CPP /nologo /W3 /Gm /GX /ZI /Od /D "WIN32" /D "_DEBUG" /D "_CONSOLE" /D "_MBCS" /YX /FD /GZ /c -# ADD CPP /nologo /MDd /W4 /WX /Gm /GX /ZI /Od /D "WIN32" /D "_DEBUG" /D "_CONSOLE" /D "_MBCS" /Yu"StdAfx.h" /FD /GZ /c +# ADD CPP /nologo /MDd /W4 /WX /Gm /GX /ZI /Od /D "_DEBUG" /D "WIN32" /D "_CONSOLE" /D "_MBCS" /D "Z7_NO_LONG_PATH" /Yu"StdAfx.h" /FD /GZ /c # ADD BASE RSC /l 0x419 /d "_DEBUG" # ADD RSC /l 0x419 /d "_DEBUG" BSC32=bscmake.exe diff --git a/CPP/7zip/Bundles/LzmaCon/makefile b/CPP/7zip/Bundles/LzmaCon/makefile index f303c31..4aef252 100644 --- a/CPP/7zip/Bundles/LzmaCon/makefile +++ b/CPP/7zip/Bundles/LzmaCon/makefile @@ -1,6 +1,7 @@ PROG = lzma.exe MY_CONSOLE = 1 +CFLAGS = $(CFLAGS) -DZ7_NO_LONG_PATH # CFLAGS = $(CFLAGS) -DZ7_ST diff --git a/CPP/7zip/Bundles/LzmaCon/makefile.gcc b/CPP/7zip/Bundles/LzmaCon/makefile.gcc index dd19a0e..67c7892 100644 --- a/CPP/7zip/Bundles/LzmaCon/makefile.gcc +++ b/CPP/7zip/Bundles/LzmaCon/makefile.gcc @@ -4,12 +4,6 @@ PROG = lzma # USE_ASM = 1 # ST_MODE = 1 -include ../../LzmaDec_gcc.mak - -LOCAL_FLAGS_ST = -MT_OBJS = - - ifdef SystemDrive IS_MINGW = 1 else @@ -19,6 +13,13 @@ IS_MINGW = 1 endif endif +include ../../LzmaDec_gcc.mak + + +LOCAL_FLAGS_ST = +MT_OBJS = + + ifdef ST_MODE LOCAL_FLAGS_ST = -DZ7_ST @@ -28,8 +29,8 @@ else MT_OBJS = \ $O/LzFindMt.o \ $O/LzFindOpt.o \ - $O/Synchronization.o \ $O/Threads.o \ + $O/Synchronization.o \ @@ -45,6 +46,9 @@ SYS_OBJS = \ $O/Registry.o \ $O/resource.o \ +LOCAL_FLAGS_SYS = \ + -DZ7_NO_LONG_PATH \ + else SYS_OBJS = \ @@ -58,6 +62,7 @@ endif LOCAL_FLAGS = \ $(LOCAL_FLAGS_ST) \ + $(LOCAL_FLAGS_SYS) \ COMMON_OBJS = \ diff --git a/CPP/7zip/Bundles/SFXCon/SFXCon.dsp b/CPP/7zip/Bundles/SFXCon/SFXCon.dsp index 1bb063d..d490932 100644 --- a/CPP/7zip/Bundles/SFXCon/SFXCon.dsp +++ b/CPP/7zip/Bundles/SFXCon/SFXCon.dsp @@ -42,7 +42,7 @@ RSC=rc.exe # PROP Ignore_Export_Lib 0 # PROP Target_Dir "" # ADD BASE CPP /nologo /W3 /GX /O2 /D "WIN32" /D "NDEBUG" /D "_CONSOLE" /D "_MBCS" /YX /FD /c -# ADD CPP /nologo /Gr /MD /W4 /WX /GX /O1 /D "NDEBUG" /D "WIN32" /D "_CONSOLE" /D "_MBCS" /D "Z7_EXTRACT_ONLY" /D "Z7_SFX" /D "Z7_NO_READ_FROM_CODER" /FAcs /Yu"StdAfx.h" /FD /c +# ADD CPP /nologo /Gr /MD /W4 /WX /GX /O1 /D "NDEBUG" /D "WIN32" /D "_CONSOLE" /D "_MBCS" /D "Z7_EXTRACT_ONLY" /D "Z7_SFX" /D "Z7_NO_READ_FROM_CODER" /D "Z7_NO_LONG_PATH" /D "Z7_NO_LARGE_PAGES" /FAcs /Yu"StdAfx.h" /FD /c # ADD BASE RSC /l 0x419 /d "NDEBUG" # ADD RSC /l 0x419 /d "NDEBUG" BSC32=bscmake.exe @@ -67,7 +67,7 @@ LINK32=link.exe # PROP Ignore_Export_Lib 0 # PROP Target_Dir "" # ADD BASE CPP /nologo /W3 /Gm /GX /ZI /Od /D "WIN32" /D "_DEBUG" /D "_CONSOLE" /D "_MBCS" /YX /FD /GZ /c -# ADD CPP /nologo /Gz /MTd /W4 /WX /Gm /GX /ZI /Od /I "..\..\..\..\\" /D "_DEBUG" /D "WIN32" /D "_CONSOLE" /D "_MBCS" /D "Z7_EXTRACT_ONLY" /D "Z7_SFX" /D "Z7_NO_READ_FROM_CODER" /Yu"StdAfx.h" /FD /GZ /c +# ADD CPP /nologo /Gz /MTd /W4 /WX /Gm /GX /ZI /Od /I "..\..\..\..\\" /D "_DEBUG" /D "WIN32" /D "_CONSOLE" /D "_MBCS" /D "Z7_EXTRACT_ONLY" /D "Z7_SFX" /D "Z7_NO_READ_FROM_CODER" /D "Z7_NO_LONG_PATH" /D "Z7_NO_LARGE_PAGES" /Yu"StdAfx.h" /FD /GZ /c # ADD BASE RSC /l 0x419 /d "_DEBUG" # ADD RSC /l 0x419 /d "_DEBUG" BSC32=bscmake.exe @@ -447,6 +447,14 @@ SOURCE=..\..\..\Windows\System.cpp SOURCE=..\..\..\Windows\System.h # End Source File +# Begin Source File + +SOURCE=..\..\..\Windows\TimeUtils.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Windows\TimeUtils.h +# End Source File # End Group # Begin Group "Common" diff --git a/CPP/7zip/Bundles/SFXCon/makefile b/CPP/7zip/Bundles/SFXCon/makefile index 51aaf51..a72e3f8 100644 --- a/CPP/7zip/Bundles/SFXCon/makefile +++ b/CPP/7zip/Bundles/SFXCon/makefile @@ -6,6 +6,8 @@ CFLAGS = $(CFLAGS) \ -DZ7_EXTRACT_ONLY \ -DZ7_NO_READ_FROM_CODER \ -DZ7_SFX \ + -DZ7_NO_LONG_PATH \ + -DZ7_NO_LARGE_PAGES \ CURRENT_OBJS = \ $O\SfxCon.obj \ @@ -43,6 +45,7 @@ WIN_OBJS = \ $O\PropVariantConv.obj \ $O\Synchronization.obj \ $O\System.obj \ + $O\TimeUtils.obj \ 7ZIP_COMMON_OBJS = \ $O\CreateCoder.obj \ diff --git a/CPP/7zip/Bundles/SFXCon/makefile.gcc b/CPP/7zip/Bundles/SFXCon/makefile.gcc index 9278502..87d78a9 100644 --- a/CPP/7zip/Bundles/SFXCon/makefile.gcc +++ b/CPP/7zip/Bundles/SFXCon/makefile.gcc @@ -47,6 +47,8 @@ LOCAL_FLAGS_SYS = ifdef IS_MINGW LOCAL_FLAGS_SYS = \ + -DZ7_NO_LONG_PATH \ + -DZ7_NO_LARGE_PAGES \ SYS_OBJS = \ $O/DLL.o \ @@ -61,11 +63,11 @@ SYS_OBJS = \ endif LOCAL_FLAGS = \ - $(LOCAL_FLAGS_ST) \ - $(LOCAL_FLAGS_SYS) \ -DZ7_EXTRACT_ONLY \ -DZ7_NO_READ_FROM_CODER \ -DZ7_SFX \ + $(LOCAL_FLAGS_ST) \ + $(LOCAL_FLAGS_SYS) \ CURRENT_OBJS = \ diff --git a/CPP/7zip/Bundles/SFXSetup/ExtractCallbackSfx.cpp b/CPP/7zip/Bundles/SFXSetup/ExtractCallbackSfx.cpp index 9d632ee..14dd2ec 100644 --- a/CPP/7zip/Bundles/SFXSetup/ExtractCallbackSfx.cpp +++ b/CPP/7zip/Bundles/SFXSetup/ExtractCallbackSfx.cpp @@ -190,7 +190,7 @@ Z7_COM7F_IMF(CExtractCallbackImp::GetStream(UInt32 index, { _outFileStreamSpec = new COutFileStream; CMyComPtr outStreamLoc(_outFileStreamSpec); - if (!_outFileStreamSpec->Create(fullProcessedPath, true)) + if (!_outFileStreamSpec->Create_ALWAYS(fullProcessedPath)) { _message = kCantOpenFile; return E_FAIL; diff --git a/CPP/7zip/Bundles/SFXSetup/SFXSetup.dsp b/CPP/7zip/Bundles/SFXSetup/SFXSetup.dsp index cf96460..09683e2 100644 --- a/CPP/7zip/Bundles/SFXSetup/SFXSetup.dsp +++ b/CPP/7zip/Bundles/SFXSetup/SFXSetup.dsp @@ -44,7 +44,7 @@ RSC=rc.exe # PROP Ignore_Export_Lib 0 # PROP Target_Dir "" # ADD BASE CPP /nologo /W3 /GX /O2 /D "WIN32" /D "NDEBUG" /D "_WINDOWS" /D "_MBCS" /YX /FD /c -# ADD CPP /nologo /Gz /MT /W4 /WX /GX /O1 /D "NDEBUG" /D "WIN32" /D "_WINDOWS" /D "_MBCS" /D "Z7_EXTRACT_ONLY" /D "Z7_NO_REGISTRY" /D "Z7_SFX" /D "Z7_NO_CRYPTO" /Yu"StdAfx.h" /FD /c +# ADD CPP /nologo /Gz /MT /W4 /WX /GX /O1 /D "NDEBUG" /D "WIN32" /D "_WINDOWS" /D "_MBCS" /D "Z7_EXTRACT_ONLY" /D "Z7_NO_REGISTRY" /D "Z7_SFX" /D "Z7_NO_CRYPTO" /D "Z7_NO_LONG_PATH" /D "Z7_NO_LARGE_PAGES" /Yu"StdAfx.h" /FD /c # ADD BASE MTL /nologo /D "NDEBUG" /mktyplib203 /win32 # ADD MTL /nologo /D "NDEBUG" /mktyplib203 /win32 # ADD BASE RSC /l 0x419 /d "NDEBUG" @@ -71,7 +71,7 @@ LINK32=link.exe # PROP Ignore_Export_Lib 0 # PROP Target_Dir "" # ADD BASE CPP /nologo /W3 /Gm /GX /ZI /Od /D "WIN32" /D "_DEBUG" /D "_WINDOWS" /D "_MBCS" /YX /FD /GZ /c -# ADD CPP /nologo /Gz /MTd /W4 /WX /Gm /GX /ZI /Od /D "_DEBUG" /D "WIN32" /D "_WINDOWS" /D "_MBCS" /D "Z7_EXTRACT_ONLY" /D "Z7_NO_REGISTRY" /D "Z7_SFX" /D "Z7_NO_CRYPTO" /Yu"StdAfx.h" /FD /GZ /c +# ADD CPP /nologo /Gz /MTd /W4 /WX /Gm /GX /ZI /Od /D "_DEBUG" /D "WIN32" /D "_WINDOWS" /D "_MBCS" /D "Z7_EXTRACT_ONLY" /D "Z7_NO_REGISTRY" /D "Z7_SFX" /D "Z7_NO_CRYPTO" /D "Z7_NO_LONG_PATH" /D "Z7_NO_LARGE_PAGES" /Yu"StdAfx.h" /FD /GZ /c # ADD BASE MTL /nologo /D "_DEBUG" /mktyplib203 /win32 # ADD MTL /nologo /D "_DEBUG" /mktyplib203 /win32 # ADD BASE RSC /l 0x419 /d "_DEBUG" @@ -98,7 +98,7 @@ LINK32=link.exe # PROP Ignore_Export_Lib 0 # PROP Target_Dir "" # ADD BASE CPP /nologo /MD /W3 /GX /O1 /D "NDEBUG" /D "WIN32" /D "_WINDOWS" /D "_MBCS" /D "Z7_EXTRACT_ONLY" /D "Z7_NO_REGISTRY" /D "Z7_SFX" /Yu"StdAfx.h" /FD /c -# ADD CPP /nologo /Gz /MD /W4 /WX /GX /O1 /D "NDEBUG" /D "WIN32" /D "_WINDOWS" /D "_MBCS" /D "Z7_EXTRACT_ONLY" /D "Z7_NO_REGISTRY" /D "Z7_SFX" /D "Z7_NO_CRYPTO" /Yu"StdAfx.h" /FD /c +# ADD CPP /nologo /Gz /MD /W4 /WX /GX /O1 /D "NDEBUG" /D "WIN32" /D "_WINDOWS" /D "_MBCS" /D "Z7_EXTRACT_ONLY" /D "Z7_NO_REGISTRY" /D "Z7_SFX" /D "Z7_NO_CRYPTO" /D "Z7_NO_LONG_PATH" /D "Z7_NO_LARGE_PAGES" /Yu"StdAfx.h" /FD /c # ADD BASE MTL /nologo /D "NDEBUG" /mktyplib203 /win32 # ADD MTL /nologo /D "NDEBUG" /mktyplib203 /win32 # ADD BASE RSC /l 0x419 /d "NDEBUG" @@ -477,6 +477,14 @@ SOURCE=..\..\..\Windows\System.h # End Source File # Begin Source File +SOURCE=..\..\..\Windows\TimeUtils.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Windows\TimeUtils.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Windows\Window.cpp # End Source File # Begin Source File diff --git a/CPP/7zip/Bundles/SFXSetup/SfxSetup.cpp b/CPP/7zip/Bundles/SFXSetup/SfxSetup.cpp index 47d7966..eb28f5d 100644 --- a/CPP/7zip/Bundles/SFXSetup/SfxSetup.cpp +++ b/CPP/7zip/Bundles/SFXSetup/SfxSetup.cpp @@ -33,6 +33,9 @@ using namespace NDir; extern HINSTANCE g_hInstance; HINSTANCE g_hInstance; +extern +bool g_DisableUserQuestions; +bool g_DisableUserQuestions; static CFSTR const kTempDirPrefix = FTEXT("7zS"); diff --git a/CPP/7zip/Bundles/SFXSetup/makefile b/CPP/7zip/Bundles/SFXSetup/makefile index ea0ece2..4f6c5fc 100644 --- a/CPP/7zip/Bundles/SFXSetup/makefile +++ b/CPP/7zip/Bundles/SFXSetup/makefile @@ -7,6 +7,8 @@ CFLAGS = $(CFLAGS) \ -DZ7_NO_READ_FROM_CODER \ -DZ7_SFX \ -DZ7_NO_CRYPTO \ + -DZ7_NO_LONG_PATH \ + -DZ7_NO_LARGE_PAGES \ CURRENT_OBJS = \ $O\SfxSetup.obj \ @@ -36,6 +38,7 @@ WIN_OBJS = \ $O\ResourceString.obj \ $O\Synchronization.obj \ $O\System.obj \ + $O\TimeUtils.obj \ $O\Window.obj \ WIN_CTRL_OBJS = \ diff --git a/CPP/7zip/Bundles/SFXWin/SFXWin.dsp b/CPP/7zip/Bundles/SFXWin/SFXWin.dsp index 65cf0bf..18db7f8 100644 --- a/CPP/7zip/Bundles/SFXWin/SFXWin.dsp +++ b/CPP/7zip/Bundles/SFXWin/SFXWin.dsp @@ -44,7 +44,7 @@ RSC=rc.exe # PROP Ignore_Export_Lib 0 # PROP Target_Dir "" # ADD BASE CPP /nologo /W3 /GX /O2 /D "WIN32" /D "NDEBUG" /D "_WINDOWS" /D "_MBCS" /YX /FD /c -# ADD CPP /nologo /Gz /MD /W4 /WX /GX /O1 /D "NDEBUG" /D "WIN32" /D "_WINDOWS" /D "_MBCS" /D "Z7_EXTRACT_ONLY" /D "Z7_NO_REGISTRY" /D "Z7_NO_READ_FROM_CODER" /D "Z7_SFX" /Yu"StdAfx.h" /FD /c +# ADD CPP /nologo /Gz /MD /W4 /WX /GX /O1 /D "NDEBUG" /D "WIN32" /D "_WINDOWS" /D "_MBCS" /D "Z7_EXTRACT_ONLY" /D "Z7_NO_REGISTRY" /D "Z7_NO_READ_FROM_CODER" /D "Z7_SFX" /D "Z7_NO_LONG_PATH" /D "Z7_NO_LARGE_PAGES" /Yu"StdAfx.h" /FD /c # ADD BASE MTL /nologo /D "NDEBUG" /mktyplib203 /win32 # ADD MTL /nologo /D "NDEBUG" /mktyplib203 /win32 # ADD BASE RSC /l 0x419 /d "NDEBUG" @@ -71,7 +71,7 @@ LINK32=link.exe # PROP Ignore_Export_Lib 0 # PROP Target_Dir "" # ADD BASE CPP /nologo /W3 /Gm /GX /ZI /Od /D "WIN32" /D "_DEBUG" /D "_WINDOWS" /D "_MBCS" /YX /FD /GZ /c -# ADD CPP /nologo /Gz /MTd /W4 /WX /Gm /GX /ZI /Od /D "_DEBUG" /D "WIN32" /D "_WINDOWS" /D "_MBCS" /D "Z7_EXTRACT_ONLY" /D "Z7_NO_REGISTRY" /D "Z7_NO_READ_FROM_CODER" /D "Z7_SFX" /Yu"StdAfx.h" /FD /GZ /c +# ADD CPP /nologo /Gz /MTd /W4 /WX /Gm /GX /ZI /Od /D "_DEBUG" /D "WIN32" /D "_WINDOWS" /D "_MBCS" /D "Z7_EXTRACT_ONLY" /D "Z7_NO_REGISTRY" /D "Z7_NO_READ_FROM_CODER" /D "Z7_SFX" /D "Z7_NO_LONG_PATH" /D "Z7_NO_LARGE_PAGES" /Yu"StdAfx.h" /FD /GZ /c # ADD BASE MTL /nologo /D "_DEBUG" /mktyplib203 /win32 # ADD MTL /nologo /D "_DEBUG" /mktyplib203 /win32 # ADD BASE RSC /l 0x419 /d "_DEBUG" @@ -98,7 +98,7 @@ LINK32=link.exe # PROP Ignore_Export_Lib 0 # PROP Target_Dir "" # ADD BASE CPP /nologo /Gz /MT /W3 /GX /O1 /I "..\..\..\\" /D "NDEBUG" /D "WIN32" /D "_WINDOWS" /D "_MBCS" /D "Z7_EXTRACT_ONLY" /D "Z7_NO_REGISTRY" /D "Z7_SFX" /Yu"StdAfx.h" /FD /c -# ADD CPP /nologo /Gz /MD /W4 /WX /GX /O1 /D "NDEBUG" /D "WIN32" /D "_WINDOWS" /D "_MBCS" /D "Z7_EXTRACT_ONLY" /D "Z7_NO_REGISTRY" /D "Z7_NO_READ_FROM_CODER" /D "Z7_SFX" /Yu"StdAfx.h" /FD /c +# ADD CPP /nologo /Gz /MD /W4 /WX /GX /O1 /D "NDEBUG" /D "WIN32" /D "_WINDOWS" /D "_MBCS" /D "Z7_EXTRACT_ONLY" /D "Z7_NO_REGISTRY" /D "Z7_NO_READ_FROM_CODER" /D "Z7_SFX" /D "Z7_NO_LONG_PATH" /D "Z7_NO_LARGE_PAGES" /Yu"StdAfx.h" /FD /c # ADD BASE MTL /nologo /D "NDEBUG" /mktyplib203 /win32 # ADD MTL /nologo /D "NDEBUG" /mktyplib203 /win32 # ADD BASE RSC /l 0x419 /d "NDEBUG" @@ -673,6 +673,14 @@ SOURCE=..\..\..\Windows\System.h # End Source File # Begin Source File +SOURCE=..\..\..\Windows\TimeUtils.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Windows\TimeUtils.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Windows\Window.cpp # End Source File # Begin Source File diff --git a/CPP/7zip/Bundles/SFXWin/SfxWin.cpp b/CPP/7zip/Bundles/SFXWin/SfxWin.cpp index 3e1880e..de2ffa4 100644 --- a/CPP/7zip/Bundles/SFXWin/SfxWin.cpp +++ b/CPP/7zip/Bundles/SFXWin/SfxWin.cpp @@ -41,11 +41,18 @@ using namespace NDir; extern HINSTANCE g_hInstance; HINSTANCE g_hInstance; +extern +bool g_DisableUserQuestions; +bool g_DisableUserQuestions; #ifndef UNDER_CE -static -DWORD g_ComCtl32Version; +#if !defined(Z7_WIN32_WINNT_MIN) || Z7_WIN32_WINNT_MIN < 0x0500 // win2000 +#define Z7_USE_DYN_ComCtl32Version +#endif + +#ifdef Z7_USE_DYN_ComCtl32Version +Z7_DIAGNOSTIC_IGNORE_CAST_FUNCTION static DWORD GetDllVersion(LPCTSTR dllName) { @@ -70,6 +77,7 @@ static DWORD GetDllVersion(LPCTSTR dllName) return dwVersion; } +#endif #endif extern @@ -86,14 +94,18 @@ static void ErrorMessageForHRESULT(HRESULT res) static int APIENTRY WinMain2() { // OleInitialize is required for ProgressBar in TaskBar. - #ifndef UNDER_CE +#ifndef UNDER_CE OleInitialize(NULL); - #endif +#endif - #ifndef UNDER_CE - g_ComCtl32Version = ::GetDllVersion(TEXT("comctl32.dll")); - g_LVN_ITEMACTIVATE_Support = (g_ComCtl32Version >= MAKELONG(71, 4)); - #endif +#ifndef UNDER_CE +#ifdef Z7_USE_DYN_ComCtl32Version + { + const DWORD g_ComCtl32Version = ::GetDllVersion(TEXT("comctl32.dll")); + g_LVN_ITEMACTIVATE_Support = (g_ComCtl32Version >= MAKELONG(71, 4)); + } +#endif +#endif UString password; bool assumeYes = false; @@ -135,6 +147,8 @@ static int APIENTRY WinMain2() } } + g_DisableUserQuestions = assumeYes; + FString path; NDLL::MyGetModuleFileName(path); diff --git a/CPP/7zip/Bundles/SFXWin/makefile b/CPP/7zip/Bundles/SFXWin/makefile index 69a8553..806bd07 100644 --- a/CPP/7zip/Bundles/SFXWin/makefile +++ b/CPP/7zip/Bundles/SFXWin/makefile @@ -6,6 +6,8 @@ CFLAGS = $(CFLAGS) \ -DZ7_EXTRACT_ONLY \ -DZ7_NO_READ_FROM_CODER \ -DZ7_SFX \ + -DZ7_NO_LONG_PATH \ + -DZ7_NO_LARGE_PAGES \ !IFDEF UNDER_CE LIBS = $(LIBS) ceshell.lib Commctrl.lib @@ -46,6 +48,7 @@ WIN_OBJS = \ $O\Shell.obj \ $O\Synchronization.obj \ $O\System.obj \ + $O\TimeUtils.obj \ $O\Window.obj \ WIN_CTRL_OBJS = \ diff --git a/CPP/7zip/Common/CWrappers.cpp b/CPP/7zip/Common/CWrappers.cpp index 346774e..d3676b7 100644 --- a/CPP/7zip/Common/CWrappers.cpp +++ b/CPP/7zip/Common/CWrappers.cpp @@ -18,6 +18,7 @@ SRes HRESULT_To_SRes(HRESULT res, SRes defaultRes) throw() case E_ABORT: return SZ_ERROR_PROGRESS; case S_FALSE: return SZ_ERROR_DATA; case E_NOTIMPL: return SZ_ERROR_UNSUPPORTED; + default: break; } return defaultRes; } @@ -32,6 +33,8 @@ HRESULT SResToHRESULT(SRes res) throw() case SZ_ERROR_DATA: case SZ_ERROR_CRC: case SZ_ERROR_INPUT_EOF: + case SZ_ERROR_ARCHIVE: + case SZ_ERROR_NO_ARCHIVE: return S_FALSE; case SZ_ERROR_MEM: return E_OUTOFMEMORY; @@ -45,6 +48,7 @@ HRESULT SResToHRESULT(SRes res) throw() // case SZ_ERROR_ARCHIVE: // case SZ_ERROR_NO_ARCHIVE: // return E_FAIL; + default: break; } if (res < 0) return res; diff --git a/CPP/7zip/Common/FilePathAutoRename.cpp b/CPP/7zip/Common/FilePathAutoRename.cpp index 1ebfd72..81172d5 100644 --- a/CPP/7zip/Common/FilePathAutoRename.cpp +++ b/CPP/7zip/Common/FilePathAutoRename.cpp @@ -19,8 +19,8 @@ static bool MakeAutoName(const FString &name, bool AutoRenamePath(FString &path) { - int dotPos = path.ReverseFind_Dot(); - int slashPos = path.ReverseFind_PathSepar(); + const int dotPos = path.ReverseFind_Dot(); + const int slashPos = path.ReverseFind_PathSepar(); FString name = path; FString extension; @@ -29,14 +29,14 @@ bool AutoRenamePath(FString &path) name.DeleteFrom((unsigned)dotPos); extension = path.Ptr((unsigned)dotPos); } - name += '_'; + name.Add_Char('_'); FString temp; - UInt32 left = 1, right = ((UInt32)1 << 30); + UInt32 left = 1, right = (UInt32)1 << 30; while (left != right) { - UInt32 mid = (left + right) / 2; + const UInt32 mid = (left + right) / 2; if (MakeAutoName(name, extension, mid, temp)) left = mid + 1; else diff --git a/CPP/7zip/Common/FileStreams.cpp b/CPP/7zip/Common/FileStreams.cpp index 4298636..f90e280 100644 --- a/CPP/7zip/Common/FileStreams.cpp +++ b/CPP/7zip/Common/FileStreams.cpp @@ -11,14 +11,26 @@ #include #include +/* +inclusion of by is deprecated since glibc 2.25. +Since glibc 2.3.3, macros have been aliases for three GNU-specific +functions: gnu_dev_makedev(), gnu_dev_major(), and gnu_dev_minor() + +Warning in GCC: +In the GNU C Library, "major" is defined by . +For historical compatibility, it is currently defined by + as well, but we plan to remove this soon. +To use "major", include directly. +If you did not intend to use a system-defined macro "major", +you should undefine it after including +*/ // for major()/minor(): +#if defined(__APPLE__) || defined(__DragonFly__) || \ + defined(BSD) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) #include -#if defined(__FreeBSD__) || defined(BSD) || defined(__APPLE__) #else -#ifndef major #include #endif -#endif #endif // _WIN32 @@ -85,6 +97,8 @@ CInFileStream::~CInFileStream() Z7_COM7F_IMF(CInFileStream::Read(void *data, UInt32 size, UInt32 *processedSize)) { + // printf("\nCInFileStream::Read size=%d, VirtPos=%8d\n", (unsigned)size, (int)VirtPos); + #ifdef Z7_FILE_STREAMS_USE_WIN_FILE #ifdef Z7_DEVICE_FILE @@ -205,6 +219,10 @@ Z7_COM7F_IMF(CInFileStream::Read(void *data, UInt32 size, UInt32 *processedSize) { const DWORD error = ::GetLastError(); +#if 0 + if (File.IsStdStream && error == ERROR_BROKEN_PIPE) + return S_OK; // end of stream +#endif if (Callback) return Callback->InFileStream_On_Error(CallbackRef, error); if (error == 0) @@ -227,13 +245,35 @@ Z7_COM7F_IMF(CStdInFileStream::Read(void *data, UInt32 size, UInt32 *processedSi #else Z7_COM7F_IMF(CStdInFileStream::Read(void *data, UInt32 size, UInt32 *processedSize)) { + // printf("\nCStdInFileStream::Read size = %d\n", (unsigned)size); #ifdef _WIN32 DWORD realProcessedSize; UInt32 sizeTemp = (1 << 20); if (sizeTemp > size) sizeTemp = size; + /* in GUI mode : GetStdHandle(STD_INPUT_HANDLE) returns NULL, + and it doesn't set LastError. */ + /* + SetLastError(0); + const HANDLE h = GetStdHandle(STD_INPUT_HANDLE); + if (!h || h == INVALID_HANDLE_VALUE) + { + if (processedSize) + *processedSize = 0; + if (GetLastError() == 0) + SetLastError(ERROR_INVALID_HANDLE); + return GetLastError_noZero_HRESULT(); + } + */ BOOL res = ::ReadFile(GetStdHandle(STD_INPUT_HANDLE), data, sizeTemp, &realProcessedSize, NULL); + + /* + printf("\nCInFileStream::Read: size=%d, processed=%8d res=%d 4rror=%3d\n", + (unsigned)size, (int)realProcessedSize, + (int)res, GetLastError()); + */ + if (processedSize) *processedSize = realProcessedSize; if (res == FALSE && GetLastError() == ERROR_BROKEN_PIPE) @@ -261,8 +301,62 @@ Z7_COM7F_IMF(CStdInFileStream::Read(void *data, UInt32 size, UInt32 *processedSi #endif + +/* +bool CreateStdInStream(CMyComPtr &str) +{ +#if 0 + CInFileStream *inStreamSpec = new CInFileStream; + CMyComPtr inStreamLoc(inStreamSpec);; + if (!inStreamSpec->OpenStdIn()) + return false; + if (!inStreamSpec->File.IsStdPipeStream) + str = inStreamLoc.Detach(); + else +#endif + str = new CStdInFileStream; + return true; +} +*/ + +#if 0 +bool CInFileStream::OpenStdIn() +{ + _info_WasLoaded = false; + // Sleep(100); + bool res = File.AttachStdIn(); + if (!res) + return false; +#if 1 + CStreamFileProps props; + if (GetProps2(&props) != S_OK) + { + // we can ignore that error + return false; + } + // we can't use Size, because Size can be set for pipe streams for some value. + // Seek() sees only current chunk in pipe buffer. + // So Seek() can move across only current unread chunk. + // But after reading that chunk. it can't move position back. + // We need safe check that shows that we can use seek (non-pipe mode) + // Is it safe check that shows that pipe mode was used? + File.IsStdPipeStream = (props.VolID == 0); + // && FILETIME_IsZero(props.CTime) + // && FILETIME_IsZero(props.ATime) + // && FILETIME_IsZero(props.MTime); +#endif + // printf("\n######## pipe=%d", (unsigned)File.IsStdPipeStream); + return true; +} +#endif + + Z7_COM7F_IMF(CInFileStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *newPosition)) { + /* + printf("\nCInFileStream::Seek seekOrigin=%d, offset=%8d, VirtPos=%8d\n", + (unsigned)seekOrigin, (int)offset, (int)VirtPos); + */ if (seekOrigin >= 3) return STG_E_INVALIDFUNCTION; @@ -463,6 +557,31 @@ Z7_COM7F_IMF(CInFileStream::ReloadProps()) _info_WasLoaded = File.GetFileInformation(&_info); if (!_info_WasLoaded) return GetLastError_HRESULT(); +#ifdef _WIN32 +#if 0 + printf( + "\ndwFileAttributes = %8x" + "\nftCreationTime = %8x" + "\nftLastAccessTime = %8x" + "\nftLastWriteTime = %8x" + "\ndwVolumeSerialNumber = %8x" + "\nnFileSizeHigh = %8x" + "\nnFileSizeLow = %8x" + "\nnNumberOfLinks = %8x" + "\nnFileIndexHigh = %8x" + "\nnFileIndexLow = %8x \n", + (unsigned)_info.dwFileAttributes, + (unsigned)_info.ftCreationTime.dwHighDateTime, + (unsigned)_info.ftLastAccessTime.dwHighDateTime, + (unsigned)_info.ftLastWriteTime.dwHighDateTime, + (unsigned)_info.dwVolumeSerialNumber, + (unsigned)_info.nFileSizeHigh, + (unsigned)_info.nFileSizeLow, + (unsigned)_info.nNumberOfLinks, + (unsigned)_info.nFileIndexHigh, + (unsigned)_info.nFileIndexLow); +#endif +#endif return S_OK; } @@ -471,6 +590,7 @@ Z7_COM7F_IMF(CInFileStream::ReloadProps()) Z7_COM7F_IMF(CInFileStream::GetProps(UInt64 *size, FILETIME *cTime, FILETIME *aTime, FILETIME *mTime, UInt32 *attrib)) { + // printf("\nCInFileStream::GetProps VirtPos = %8d\n", (int)VirtPos); if (!_info_WasLoaded) { RINOK(ReloadProps()) @@ -495,6 +615,7 @@ Z7_COM7F_IMF(CInFileStream::GetProps(UInt64 *size, FILETIME *cTime, FILETIME *aT Z7_COM7F_IMF(CInFileStream::GetProps2(CStreamFileProps *props)) { + // printf("\nCInFileStream::GetProps2 VirtPos = %8d\n", (int)VirtPos); if (!_info_WasLoaded) { RINOK(ReloadProps()) @@ -535,6 +656,7 @@ Z7_COM7F_IMF(CInFileStream::GetProps2(CStreamFileProps *props)) Z7_COM7F_IMF(CInFileStream::GetProperty(PROPID propID, PROPVARIANT *value)) { + // printf("\nCInFileStream::GetProperty VirtPos = %8d propID = %3d\n", (int)VirtPos, propID); if (!_info_WasLoaded) { RINOK(ReloadProps()) @@ -648,6 +770,7 @@ Z7_COM7F_IMF(CInFileStream::GetProperty(PROPID propID, PROPVARIANT *value)) } break; } + default: break; } } prop.Detach(value); diff --git a/CPP/7zip/Common/FileStreams.h b/CPP/7zip/Common/FileStreams.h index 7e1b086..212d4f0 100644 --- a/CPP/7zip/Common/FileStreams.h +++ b/CPP/7zip/Common/FileStreams.h @@ -46,8 +46,9 @@ Z7_class_final(CInFileStream) : public IStreamGetProp, public CMyUnknownImp { - Z7_COM_UNKNOWN_IMP_5( + Z7_COM_UNKNOWN_IMP_6( IInStream, + ISequentialInStream, IStreamGetSize, IStreamGetProps, IStreamGetProps2, @@ -108,6 +109,10 @@ Z7_class_final(CInFileStream) : { return File.GetLength(length); } + +#if 0 + bool OpenStdIn(); +#endif bool Open(CFSTR fileName) { @@ -122,6 +127,7 @@ Z7_class_final(CInFileStream) : } }; +// bool CreateStdInStream(CMyComPtr &str); Z7_CLASS_IMP_NOQIB_1( CStdInFileStream @@ -139,15 +145,28 @@ Z7_CLASS_IMP_COM_1( NWindows::NFile::NIO::COutFile File; - bool Create(CFSTR fileName, bool createAlways) + bool Create_NEW(CFSTR fileName) + { + ProcessedSize = 0; + return File.Create_NEW(fileName); + } + + bool Create_ALWAYS(CFSTR fileName) { ProcessedSize = 0; - return File.Create(fileName, createAlways); + return File.Create_ALWAYS(fileName); } - bool Open(CFSTR fileName, DWORD creationDisposition) + + bool Open_EXISTING(CFSTR fileName) + { + ProcessedSize = 0; + return File.Open_EXISTING(fileName); + } + + bool Create_ALWAYS_or_Open_ALWAYS(CFSTR fileName, bool createAlways) { ProcessedSize = 0; - return File.Open(fileName, creationDisposition); + return File.Create_ALWAYS_or_Open_ALWAYS(fileName, createAlways); } HRESULT Close(); diff --git a/CPP/7zip/Common/InBuffer.cpp b/CPP/7zip/Common/InBuffer.cpp index b0f222c..ae004fb 100644 --- a/CPP/7zip/Common/InBuffer.cpp +++ b/CPP/7zip/Common/InBuffer.cpp @@ -96,6 +96,24 @@ Byte CInBufferBase::ReadByte_FromNewBlock() return *_buf++; } +size_t CInBufferBase::ReadBytesPart(Byte *buf, size_t size) +{ + if (size == 0) + return 0; + size_t rem = (size_t)(_bufLim - _buf); + if (rem == 0) + { + if (!ReadBlock()) + return 0; + rem = (size_t)(_bufLim - _buf); + } + if (size > rem) + size = rem; + memcpy(buf, _buf, size); + _buf += size; + return size; +} + size_t CInBufferBase::ReadBytes(Byte *buf, size_t size) { size_t num = 0; diff --git a/CPP/7zip/Common/InBuffer.h b/CPP/7zip/Common/InBuffer.h index 3aaf797..a8ccb40 100644 --- a/CPP/7zip/Common/InBuffer.h +++ b/CPP/7zip/Common/InBuffer.h @@ -51,6 +51,7 @@ class CInBufferBase bool WasFinished() const { return _wasFinished; } void SetStream(ISequentialInStream *stream) { _stream = stream; } + void ClearStreamPtr() { _stream = NULL; } void SetBuf(Byte *buf, size_t bufSize, size_t end, size_t pos) { @@ -94,6 +95,7 @@ class CInBufferBase return *_buf++; } + size_t ReadBytesPart(Byte *buf, size_t size); size_t ReadBytes(Byte *buf, size_t size); size_t Skip(size_t size); }; diff --git a/CPP/7zip/Common/MemBlocks.cpp b/CPP/7zip/Common/MemBlocks.cpp index aa68244..6165c7e 100644 --- a/CPP/7zip/Common/MemBlocks.cpp +++ b/CPP/7zip/Common/MemBlocks.cpp @@ -57,7 +57,7 @@ void CMemBlockManager::FreeBlock(void *p) // #include -HRes CMemBlockManagerMt::AllocateSpace(size_t numBlocks, size_t numNoLockBlocks) +HRESULT CMemBlockManagerMt::AllocateSpace(size_t numBlocks, size_t numNoLockBlocks) { if (numNoLockBlocks > numBlocks) return E_INVALIDARG; @@ -87,7 +87,7 @@ HRes CMemBlockManagerMt::AllocateSpace(size_t numBlocks, size_t numNoLockBlocks) } -HRes CMemBlockManagerMt::AllocateSpaceAlways(size_t desiredNumberOfBlocks, size_t numNoLockBlocks) +HRESULT CMemBlockManagerMt::AllocateSpaceAlways(size_t desiredNumberOfBlocks, size_t numNoLockBlocks) { // desiredNumberOfBlocks = 0; // for debug if (numNoLockBlocks > desiredNumberOfBlocks) @@ -95,7 +95,7 @@ HRes CMemBlockManagerMt::AllocateSpaceAlways(size_t desiredNumberOfBlocks, size_ for (;;) { // if (desiredNumberOfBlocks == 0) return E_OUTOFMEMORY; - HRes hres = AllocateSpace(desiredNumberOfBlocks, numNoLockBlocks); + const HRESULT hres = AllocateSpace(desiredNumberOfBlocks, numNoLockBlocks); if (hres != E_OUTOFMEMORY) return hres; if (desiredNumberOfBlocks == numNoLockBlocks) diff --git a/CPP/7zip/Common/MemBlocks.h b/CPP/7zip/Common/MemBlocks.h index 7c34f21..201f3bc 100644 --- a/CPP/7zip/Common/MemBlocks.h +++ b/CPP/7zip/Common/MemBlocks.h @@ -36,8 +36,8 @@ class CMemBlockManagerMt: public CMemBlockManager CMemBlockManagerMt(size_t blockSize = (1 << 20)): CMemBlockManager(blockSize) {} ~CMemBlockManagerMt() { FreeSpace(); } - HRes AllocateSpace(size_t numBlocks, size_t numNoLockBlocks); - HRes AllocateSpaceAlways(size_t desiredNumberOfBlocks, size_t numNoLockBlocks = 0); + HRESULT AllocateSpace(size_t numBlocks, size_t numNoLockBlocks); + HRESULT AllocateSpaceAlways(size_t desiredNumberOfBlocks, size_t numNoLockBlocks = 0); void FreeSpace(); void *AllocateBlock(); void FreeBlock(void *p, bool lockMode = true); @@ -65,7 +65,7 @@ struct CMemLockBlocks: public CMemBlocks CMemLockBlocks(): LockMode(true) {} void Free(CMemBlockManagerMt *memManager); void FreeBlock(unsigned index, CMemBlockManagerMt *memManager); - // HRes SwitchToNoLockMode(CMemBlockManagerMt *memManager); + // HRESULT SwitchToNoLockMode(CMemBlockManagerMt *memManager); void Detach(CMemLockBlocks &blocks, CMemBlockManagerMt *memManager); }; diff --git a/CPP/7zip/Common/MethodProps.cpp b/CPP/7zip/Common/MethodProps.cpp index fe60f61..d87884c 100644 --- a/CPP/7zip/Common/MethodProps.cpp +++ b/CPP/7zip/Common/MethodProps.cpp @@ -60,6 +60,7 @@ HRESULT PROPVARIANT_to_bool(const PROPVARIANT &prop, bool &dest) case VT_EMPTY: dest = true; return S_OK; case VT_BOOL: dest = (prop.boolVal != VARIANT_FALSE); return S_OK; case VT_BSTR: return StringToBool(prop.bstrVal, dest) ? S_OK : E_INVALIDARG; + default: break; } return E_INVALIDARG; } @@ -541,6 +542,7 @@ static bool IsLogSizeProp(PROPID propid) */ // case NCoderPropID::kReduceSize: return true; + default: break; } return false; } diff --git a/CPP/7zip/Common/MultiOutStream.cpp b/CPP/7zip/Common/MultiOutStream.cpp index 8efb977..1954311 100644 --- a/CPP/7zip/Common/MultiOutStream.cpp +++ b/CPP/7zip/Common/MultiOutStream.cpp @@ -183,7 +183,7 @@ bool CMultiOutStream::IsRestricted_for_Close(unsigned index) const FString CMultiOutStream::GetFilePath(unsigned index) { FString name; - name.Add_UInt32(index + 1); + name.Add_UInt32((UInt32)(index + 1)); while (name.Len() < 3) name.InsertAtFront(FTEXT('0')); name.Insert(0, Prefix); @@ -209,7 +209,7 @@ HRESULT CMultiOutStream::CloseStream(unsigned index) // we close stream and delete file, but we still keep item in Streams[] vector HRESULT CMultiOutStream::CloseStream_and_DeleteFile(unsigned index) { - PRF(printf("\n====== %u, CloseStream_AndDelete \n", index)); + PRF(printf("\n====== %u, CloseStream_AndDelete \n", index)) RINOK(CloseStream(index)) FString path = GetFilePath(index); path += Streams[index].Postfix; @@ -223,7 +223,7 @@ HRESULT CMultiOutStream::CloseStream_and_DeleteFile(unsigned index) HRESULT CMultiOutStream::CloseStream_and_FinalRename(unsigned index) { - PRF(printf("\n====== %u, CloseStream_and_FinalRename \n", index)); + PRF(printf("\n====== %u, CloseStream_and_FinalRename \n", index)) CVolStream &s = Streams[index]; // HRESULT res = S_OK; bool mtime_WasSet = false; @@ -261,6 +261,8 @@ HRESULT CMultiOutStream::CloseStream_and_FinalRename(unsigned index) HRESULT CMultiOutStream::PrepareToOpenNew() { + PRF(printf("PrepareToOpenNew NumListItems =%u, NumOpenFiles_AllowedMax = %u \n", NumListItems, NumOpenFiles_AllowedMax)) + if (NumListItems < NumOpenFiles_AllowedMax) return S_OK; /* when we create zip archive: in most cases we need only starting @@ -270,7 +272,7 @@ HRESULT CMultiOutStream::PrepareToOpenNew() const int index = Head; if (index == -1) return E_FAIL; - PRF(printf("\n== %u, PrepareToOpenNew::CloseStream, NumListItems =%u \n", index, NumListItems)); + PRF(printf("\n== %u, PrepareToOpenNew::CloseStream, NumListItems =%u \n", index, NumListItems)) /* we don't expect non-restricted stream here in normal cases (if _restrict_Global was not changed). if there was non-restricted stream, it should be closed before */ // if (!IsRestricted_for_Close(index)) return CloseStream_and_FinalRename(index); @@ -280,7 +282,7 @@ HRESULT CMultiOutStream::PrepareToOpenNew() HRESULT CMultiOutStream::CreateNewStream(UInt64 newSize) { - PRF(printf("\n== %u, CreateNewStream, size =%u \n", Streams.Size(), (unsigned)newSize)); + PRF(printf("\n== %u, CreateNewStream, size =%u \n", Streams.Size(), (unsigned)newSize)) if (Streams.Size() >= k_NumVols_MAX) return E_INVALIDARG; // E_OUTOFMEMORY @@ -358,7 +360,7 @@ HRESULT CMultiOutStream::CreateStreams_If_Required(unsigned streamIndex) HRESULT CMultiOutStream::ReOpenStream(unsigned streamIndex) { - PRF(printf("\n====== %u, ReOpenStream \n", streamIndex)); + PRF(printf("\n====== %u, ReOpenStream \n", streamIndex)) RINOK(PrepareToOpenNew()) CVolStream &s = Streams[streamIndex]; @@ -370,7 +372,7 @@ HRESULT CMultiOutStream::ReOpenStream(unsigned streamIndex) s.Pos = 0; HRESULT hres; - if (s.StreamSpec->Open(path, OPEN_EXISTING)) + if (s.StreamSpec->Open_EXISTING(path)) { if (s.Postfix.IsEmpty()) { @@ -384,6 +386,7 @@ HRESULT CMultiOutStream::ReOpenStream(unsigned streamIndex) { if (realSize == s.RealSize) { + PRF(printf("\n ReOpenStream OK realSize = %u\n", (unsigned)realSize)) InsertToLinkedList(streamIndex); return S_OK; } @@ -412,7 +415,7 @@ HRESULT CMultiOutStream::OptReOpen_and_SetSize(unsigned index, UInt64 size) { RINOK(ReOpenStream(index)) } - PRF(printf("\n== %u, OptReOpen_and_SetSize, size =%u RealSize = %u\n", index, (unsigned)size, (unsigned)s.RealSize)); + PRF(printf("\n== %u, OptReOpen_and_SetSize, size =%u RealSize = %u\n", index, (unsigned)size, (unsigned)s.RealSize)) // comment it to debug tail after data return s.SetSize2(size); } @@ -440,7 +443,7 @@ Note: we don't remove zero sized first volume, if (_length == 0) HRESULT CMultiOutStream::Normalize_finalMode(bool finalMode) { - PRF(printf("\n== Normalize_finalMode: _length =%d \n", (unsigned)_length)); + PRF(printf("\n== Normalize_finalMode: _length =%d \n", (unsigned)_length)) unsigned i = Streams.Size(); @@ -579,7 +582,7 @@ Z7_COM7F_IMF(CMultiOutStream::SetSize(UInt64 newSize)) else if (newSize < _restrict_Global) _restrict_Global = newSize; - PRF(printf("\n== SetSize, size =%u \n", (unsigned)newSize)); + PRF(printf("\n== CMultiOutStream::SetSize, size =%u \n", (unsigned)newSize)) _length = newSize; return Normalize_finalMode(false); @@ -596,6 +599,9 @@ Z7_COM7F_IMF(CMultiOutStream::Write(const void *data, UInt32 size, UInt32 *proce if (size == 0) return S_OK; + PRF(printf("\n -- CMultiOutStream::Write() : _absPos = %6u, size =%6u \n", + (unsigned)_absPos, (unsigned)size)) + if (_absPos > _length) { // it create data only up to _absPos. @@ -648,7 +654,7 @@ Z7_COM7F_IMF(CMultiOutStream::Write(const void *data, UInt32 size, UInt32 *proce CVolStream &s = Streams[_streamIndex]; PRF(printf("\n%d, == Write : Pos = %u, RealSize = %u size =%u \n", - _streamIndex, (unsigned)s.Pos, (unsigned)s.RealSize, size)); + _streamIndex, (unsigned)s.Pos, (unsigned)s.RealSize, size)) if (!s.Stream) { @@ -712,7 +718,7 @@ Z7_COM7F_IMF(CMultiOutStream::Write(const void *data, UInt32 size, UInt32 *proce Z7_COM7F_IMF(CMultiOutStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *newPosition)) { - PRF(printf("\n-- Seek seekOrigin=%u Seek =%u\n", seekOrigin, (unsigned)offset)); + PRF(printf("\n-- CMultiOutStream::Seek seekOrigin=%u Seek =%u\n", seekOrigin, (unsigned)offset)) switch (seekOrigin) { @@ -753,7 +759,7 @@ unsigned CMultiOutStream::GetStreamIndex_for_Offset(UInt64 offset, UInt64 &relOf const UInt64 size = Sizes[last]; const UInt64 v = offset / size; if (v >= ((UInt32)(Int32)-1) - last) - return (UInt32)(Int32)-1; // saturation + return (unsigned)(int)-1; // saturation relOffset = offset - (unsigned)v * size; return last + (unsigned)(v); } @@ -765,7 +771,7 @@ Z7_COM7F_IMF(CMultiOutStream::SetRestriction(UInt64 begin, UInt64 end)) // begin = end = 0; // for debug - PRF(printf("\n==================== CMultiOutStream::SetRestriction %u, %u\n", (unsigned)begin, (unsigned)end)); + PRF(printf("\n==================== CMultiOutStream::SetRestriction %u, %u\n", (unsigned)begin, (unsigned)end)) if (begin > end) { // these value are FAILED values. diff --git a/CPP/7zip/Common/OutBuffer.h b/CPP/7zip/Common/OutBuffer.h index cef7d50..88f5787 100644 --- a/CPP/7zip/Common/OutBuffer.h +++ b/CPP/7zip/Common/OutBuffer.h @@ -60,6 +60,48 @@ class COutBuffer WriteByte(((const Byte *)data)[i]); } + Byte *GetOutBuffer(size_t &avail) + { + const UInt32 pos = _pos; + avail = (size_t)(_limitPos - pos); + return _buf + pos; + } + + void SkipWrittenBytes(size_t num) + { + const UInt32 pos = _pos; + const UInt32 rem = _limitPos - pos; + if (rem > num) + { + _pos = pos + (UInt32)num; + return; + } + // (rem <= num) + // the caller must not call it with (rem < num) + // so (rem == num) + _pos = _limitPos; + FlushWithCheck(); + } + /* + void WriteBytesBig(const void *data, size_t size) + { + while (size) + { + UInt32 pos = _pos; + UInt32 rem = _limitPos - pos; + if (rem > size) + { + _pos = pos + size; + memcpy(_buf + pos, data, size); + return; + } + memcpy(_buf + pos, data, rem); + _pos = pos + rem; + FlushWithCheck(); + } + } + */ + UInt64 GetProcessedSize() const throw(); }; diff --git a/CPP/7zip/Common/OutMemStream.h b/CPP/7zip/Common/OutMemStream.h index 7fc2dbd..a58fb91 100644 --- a/CPP/7zip/Common/OutMemStream.h +++ b/CPP/7zip/Common/OutMemStream.h @@ -33,7 +33,7 @@ Z7_CLASS_IMP_NOQIB_1( public: - HRes CreateEvents(SYNC_PARAM_DECL(synchro)) + HRESULT CreateEvents(SYNC_PARAM_DECL(synchro)) { WRes wres = StopWritingEvent.CreateIfNotCreated_Reset(SYNC_WFMO(synchro)); if (wres == 0) diff --git a/CPP/7zip/Compress/BZip2Crc.cpp b/CPP/7zip/Compress/BZip2Crc.cpp index 9639c7b..1c62253 100644 --- a/CPP/7zip/Compress/BZip2Crc.cpp +++ b/CPP/7zip/Compress/BZip2Crc.cpp @@ -4,6 +4,7 @@ #include "BZip2Crc.h" +MY_ALIGN(64) UInt32 CBZip2Crc::Table[256]; static const UInt32 kBZip2CrcPoly = 0x04c11db7; /* AUTODIN II, Ethernet, & FDDI */ @@ -12,7 +13,7 @@ void CBZip2Crc::InitTable() { for (UInt32 i = 0; i < 256; i++) { - UInt32 r = (i << 24); + UInt32 r = i << 24; for (unsigned j = 0; j < 8; j++) r = (r << 1) ^ (kBZip2CrcPoly & ((UInt32)0 - (r >> 31))); Table[i] = r; diff --git a/CPP/7zip/Compress/BZip2Decoder.cpp b/CPP/7zip/Compress/BZip2Decoder.cpp index b28da5f..f44faa2 100644 --- a/CPP/7zip/Compress/BZip2Decoder.cpp +++ b/CPP/7zip/Compress/BZip2Decoder.cpp @@ -42,7 +42,7 @@ static const size_t kOutBufSize = (size_t)1 << 20; static const UInt32 kProgressStep = (UInt32)1 << 16; - +MY_ALIGN(64) static const UInt16 kRandNums[512] = { 619, 720, 127, 481, 931, 816, 813, 233, 566, 247, 985, 724, 205, 454, 863, 491, 741, 242, 949, 214, @@ -120,18 +120,18 @@ enum EState }; -#define UPDATE_VAL_2(val) { \ - val |= (UInt32)(*_buf) << (24 - _numBits); \ - _numBits += 8; \ +#define UPDATE_VAL_2(val, num_bits) { \ + val |= (UInt32)(*_buf) << (24 - num_bits); \ + num_bits += 8; \ _buf++; \ } -#define UPDATE_VAL UPDATE_VAL_2(VAL) +#define UPDATE_VAL UPDATE_VAL_2(VAL, NUM_BITS) #define READ_BITS(res, num) { \ while (_numBits < num) { \ if (_buf == _lim) return SZ_OK; \ - UPDATE_VAL_2(_value) } \ + UPDATE_VAL_2(_value, _numBits) } \ res = _value >> (32 - num); \ _value <<= num; \ _numBits -= num; \ @@ -140,7 +140,7 @@ enum EState #define READ_BITS_8(res, num) { \ if (_numBits < num) { \ if (_buf == _lim) return SZ_OK; \ - UPDATE_VAL_2(_value) } \ + UPDATE_VAL_2(_value, _numBits) } \ res = _value >> (32 - num); \ _value <<= num; \ _numBits -= num; \ @@ -151,16 +151,20 @@ enum EState #define VAL _value2 +// #define NUM_BITS _numBits2 +#define NUM_BITS _numBits #define BLOCK_SIZE blockSize2 #define RUN_COUNTER runCounter2 #define LOAD_LOCAL \ UInt32 VAL = this->_value; \ + /* unsigned NUM_BITS = this->_numBits; */ \ UInt32 BLOCK_SIZE = this->blockSize; \ UInt32 RUN_COUNTER = this->runCounter; \ #define SAVE_LOCAL \ this->_value = VAL; \ + /* this->_numBits = NUM_BITS; */ \ this->blockSize = BLOCK_SIZE; \ this->runCounter = RUN_COUNTER; \ @@ -364,8 +368,8 @@ SRes CBase::ReadBlock2() if (++state4 >= numTables) return SZ_ERROR_DATA; } - UInt32 tmp = (state2 >> (kMtfBits * state4)) & kMtfMask; - UInt32 mask = ((UInt32)1 << ((state4 + 1) * kMtfBits)) - 1; + const UInt32 tmp = (state2 >> (kMtfBits * state4)) & kMtfMask; + const UInt32 mask = ((UInt32)1 << ((state4 + 1) * kMtfBits)) - 1; state4 = 0; state2 = ((state2 << kMtfBits) & mask) | (state2 & ~mask) | tmp; // 20.01: here we keep compatibility with bzip2-1.0.8 decoder: @@ -424,15 +428,11 @@ SRes CBase::ReadBlock2() state5 = 0; } - // 19.03: we use Build() instead of BuildFull() to support lbzip2 archives + // 19.03: we use non-full Build() to support lbzip2 archives. // lbzip2 2.5 can produce dummy tree, where lens[i] = kMaxHuffmanLen - // BuildFull() returns error for such tree for (unsigned i = state4; i < kMaxAlphaSize; i++) lens[i] = 0; - if (!huffs[state2].Build(lens)) - /* - if (!huffs[state2].BuildFull(lens, state4)) - */ + if (!huffs[state2].Build(lens)) // k_BuildMode_Partial return SZ_ERROR_DATA; state3 = 0; } @@ -462,7 +462,7 @@ SRes CBase::ReadBlock2() { LOAD_LOCAL - const CHuffmanDecoder *huff = &huffs[selectors[groupIndex]]; + const CHuffmanDecoder *huf = &huffs[selectors[groupIndex]]; for (;;) { @@ -470,58 +470,38 @@ SRes CBase::ReadBlock2() { if (++groupIndex >= numSelectors) return SZ_ERROR_DATA; - huff = &huffs[selectors[groupIndex]]; + huf = &huffs[selectors[groupIndex]]; groupSize = kGroupSize; } - if (_numBits <= 8 && - _buf != _lim) { UPDATE_VAL - if (_buf != _lim) { UPDATE_VAL - if (_buf != _lim) { UPDATE_VAL }}} - - UInt32 sym; - UInt32 val = VAL >> (32 - kMaxHuffmanLen); - if (val >= huff->_limits[kNumTableBits]) - { - if (_numBits <= kMaxHuffmanLen && _buf != _lim) { UPDATE_VAL - if (_numBits <= kMaxHuffmanLen && _buf != _lim) { UPDATE_VAL }} - - val = VAL >> (32 - kMaxHuffmanLen); - unsigned len; - for (len = kNumTableBits + 1; val >= huff->_limits[len]; len++); - - // 19.03: we use that check to support partial trees created Build() for lbzip2 archives - if (len > kNumBitsMax) - return SZ_ERROR_DATA; // that check is required, if NHuffman::Build() was used instead of BuildFull() - - if (_numBits < len) - { - SAVE_LOCAL - return SZ_OK; - } - sym = huff->_symbols[huff->_poses[len] + ((val - huff->_limits[(size_t)len - 1]) >> (kNumBitsMax - len))]; - VAL <<= len; - _numBits -= len; - } - else - { - sym = huff->_lens[val >> (kMaxHuffmanLen - kNumTableBits)]; - unsigned len = (sym & NHuffman::kPairLenMask); - sym >>= NHuffman::kNumPairLenBits; - if (_numBits < len) - { - SAVE_LOCAL - return SZ_OK; - } - VAL <<= len; - _numBits -= len; + if (NUM_BITS < kMaxHuffmanLen && _buf != _lim) { UPDATE_VAL + if (NUM_BITS < kMaxHuffmanLen && _buf != _lim) { UPDATE_VAL + if (NUM_BITS < kMaxHuffmanLen && _buf != _lim) { UPDATE_VAL }}} + + unsigned sym; + + #define MOV_POS(bs, len) \ + { \ + if (NUM_BITS < len) \ + { \ + SAVE_LOCAL \ + return SZ_OK; \ + } \ + VAL <<= len; \ + NUM_BITS -= (unsigned)len; \ } + Z7_HUFF_DECODE_VAL_IN_HIGH32(sym, huf, kMaxHuffmanLen, kNumTableBits, + VAL, + Z7_HUFF_DECODE_ERROR_SYM_CHECK_YES, + { return SZ_ERROR_DATA; }, + MOV_POS, {}, bs) + groupSize--; if (sym < 2) { - RUN_COUNTER += ((UInt32)(sym + 1) << runPower); + RUN_COUNTER += (UInt32)(sym + 1) << runPower; runPower++; if (blockSizeMax - BLOCK_SIZE < RUN_COUNTER) return SZ_ERROR_DATA; diff --git a/CPP/7zip/Compress/BZip2Decoder.h b/CPP/7zip/Compress/BZip2Decoder.h index a8ef700..4adc564 100644 --- a/CPP/7zip/Compress/BZip2Decoder.h +++ b/CPP/7zip/Compress/BZip2Decoder.h @@ -27,7 +27,6 @@ bool IsEndSig(const Byte *p) throw(); bool IsBlockSig(const Byte *p) throw(); const unsigned kNumTableBits = 9; -const unsigned kNumBitsMax = kMaxHuffmanLen; typedef NHuffman::CDecoder CHuffmanDecoder; @@ -99,7 +98,7 @@ struct CBase: public CBitDecoder UInt32 blockSizeMax; unsigned state; - unsigned state2; + UInt32 state2; unsigned state3; unsigned state4; unsigned state5; diff --git a/CPP/7zip/Compress/Bcj2Coder.cpp b/CPP/7zip/Compress/Bcj2Coder.cpp index 27e78b0..4fa42a5 100644 --- a/CPP/7zip/Compress/Bcj2Coder.cpp +++ b/CPP/7zip/Compress/Bcj2Coder.cpp @@ -506,7 +506,7 @@ void CBaseDecoder::ReadInStream(ISequentialInStream *inStream) if (BCJ2_IS_32BIT_STREAM(state)) { - const unsigned extra = ((unsigned)total & 3); + const unsigned extra = (unsigned)total & 3; _extraSizes[state] = extra; if (total < 4) { @@ -514,7 +514,7 @@ void CBaseDecoder::ReadInStream(ISequentialInStream *inStream) _readRes[state] = S_FALSE; // actually it's stream error. So maybe we need another error code. return; } - total -= extra; + total -= (UInt32)extra; } dec.lims[state] += total; // = _bufs[state] + total; @@ -655,6 +655,20 @@ Z7_COM7F_IMF(CDecoder::Code( return S_FALSE; } } + + /* v23.02: we call Read(0) for BCJ2_STREAM_CALL and BCJ2_STREAM_JUMP streams, + if there were no Read() calls for such stream. + So the handlers of these input streams objects can do + Init/Flushing even for case when stream is empty: + */ + for (unsigned i = BCJ2_STREAM_CALL; i < BCJ2_STREAM_CALL + 2; i++) + { + if (_readSizes[i]) + continue; + Byte b; + UInt32 processed; + RINOK(inStreams[i]->Read(&b, 0, &processed)) + } } return S_OK; diff --git a/CPP/7zip/Compress/BitlDecoder.cpp b/CPP/7zip/Compress/BitlDecoder.cpp index 876e649..bf1e5ba 100644 --- a/CPP/7zip/Compress/BitlDecoder.cpp +++ b/CPP/7zip/Compress/BitlDecoder.cpp @@ -6,20 +6,27 @@ namespace NBitl { -Byte kInvertTable[256]; +#if defined(Z7_BITL_USE_REVERSE_BITS_TABLE) + +MY_ALIGN(64) +Byte kReverseTable[256]; static -struct CInverterTableInitializer +struct CReverseerTableInitializer { - CInverterTableInitializer() + CReverseerTableInitializer() { for (unsigned i = 0; i < 256; i++) { - unsigned x = ((i & 0x55) << 1) | ((i & 0xAA) >> 1); - x = ((x & 0x33) << 2) | ((x & 0xCC) >> 2); - kInvertTable[i] = (Byte)(((x & 0x0F) << 4) | ((x & 0xF0) >> 4)); + unsigned + x = ((i & 0x55) << 1) | ((i >> 1) & 0x55); + x = ((x & 0x33) << 2) | ((x >> 2) & 0x33); + kReverseTable[i] = (Byte)((x << 4) | (x >> 4)); } } -} g_InverterTableInitializer; +} g_ReverseerTableInitializer; +#elif 0 +unsigned ReverseBits8test(unsigned i) { return ReverseBits8(i); } +#endif } diff --git a/CPP/7zip/Compress/BitlDecoder.h b/CPP/7zip/Compress/BitlDecoder.h index 789ad1f..465c4d3 100644 --- a/CPP/7zip/Compress/BitlDecoder.h +++ b/CPP/7zip/Compress/BitlDecoder.h @@ -3,6 +3,8 @@ #ifndef ZIP7_INC_BITL_DECODER_H #define ZIP7_INC_BITL_DECODER_H +#include "../../../C/CpuArch.h" + #include "../IStream.h" namespace NBitl { @@ -10,10 +12,50 @@ namespace NBitl { const unsigned kNumBigValueBits = 8 * 4; const unsigned kNumValueBytes = 3; const unsigned kNumValueBits = 8 * kNumValueBytes; - const UInt32 kMask = (1 << kNumValueBits) - 1; -extern Byte kInvertTable[256]; +#if !defined(Z7_BITL_USE_REVERSE_BITS_TABLE) +#if 1 && defined(MY_CPU_ARM_OR_ARM64) \ + && (defined(MY_CPU_ARM64) || defined(__ARM_ARCH_6T2__) \ + || defined(__ARM_ARCH) && (__ARM_ARCH >= 7)) \ + && (defined(__GNUC__) && (__GNUC__ >= 4) \ + || defined(__clang__) && (__clang_major__ >= 4)) + #define Z7_BITL_USE_REVERSE_BITS_INSTRUCTION +#elif 1 + #define Z7_BITL_USE_REVERSE_BITS_TABLE +#endif +#endif + +#if defined(Z7_BITL_USE_REVERSE_BITS_TABLE) +extern Byte kReverseTable[256]; +#endif + +inline unsigned ReverseBits8(unsigned i) +{ +#if defined(Z7_BITL_USE_REVERSE_BITS_TABLE) + return kReverseTable[i]; +#elif defined(Z7_BITL_USE_REVERSE_BITS_INSTRUCTION) + // rbit is available in ARMv6T2 and above + asm ("rbit " +#if defined(MY_CPU_ARM) + "%0,%0" // it uses default register size, + // but we need 32-bit register here. + // we must use it only if default register size is 32-bit. + // it will work incorrectly for ARM64. +#else + "%w0,%w0" // it uses 32-bit registers in ARM64. + // compiler for (MY_CPU_ARM) can't compile it. +#endif + : "+r" (i)); + return i >> 24; +#else + unsigned + x = ((i & 0x55) << 1) | ((i >> 1) & 0x55); + x = ((x & 0x33) << 2) | ((x >> 2) & 0x33); + return ((x & 0x0f) << 4) | (x >> 4); +#endif +} + /* TInByte must support "Extra Bytes" (bytes that can be read after the end of stream TInByte::ReadByte() returns 0xFF after the end of stream @@ -31,6 +73,7 @@ class CBaseDecoder public: bool Create(UInt32 bufSize) { return _stream.Create(bufSize); } void SetStream(ISequentialInStream *inStream) { _stream.SetStream(inStream); } + void ClearStreamPtr() { _stream.ClearStreamPtr(); } void Init() { _stream.Init(); @@ -107,9 +150,9 @@ class CDecoder: public CBaseDecoder { for (; this->_bitPos >= 8; this->_bitPos -= 8) { - Byte b = this->_stream.ReadByte(); + const unsigned b = this->_stream.ReadByte(); _normalValue = ((UInt32)b << (kNumBigValueBits - this->_bitPos)) | _normalValue; - this->_value = (this->_value << 8) | kInvertTable[b]; + this->_value = (this->_value << 8) | ReverseBits8(b); } } @@ -119,11 +162,18 @@ class CDecoder: public CBaseDecoder Normalize(); return ((this->_value >> (8 - this->_bitPos)) & kMask) >> (kNumValueBits - numBits); } + + Z7_FORCE_INLINE + UInt32 GetValue_InHigh32bits() + { + Normalize(); + return this->_value << this->_bitPos; + } Z7_FORCE_INLINE - void MovePos(unsigned numBits) + void MovePos(size_t numBits) { - this->_bitPos += numBits; + this->_bitPos += (unsigned)numBits; _normalValue >>= numBits; } @@ -140,6 +190,9 @@ class CDecoder: public CBaseDecoder Z7_FORCE_INLINE Byte ReadDirectByte() { return this->_stream.ReadByte(); } + + Z7_FORCE_INLINE + size_t ReadDirectBytesPart(Byte *buf, size_t size) { return this->_stream.ReadBytesPart(buf, size); } Z7_FORCE_INLINE Byte ReadAlignedByte() diff --git a/CPP/7zip/Compress/BitmDecoder.h b/CPP/7zip/Compress/BitmDecoder.h index 199a228..21c83db 100644 --- a/CPP/7zip/Compress/BitmDecoder.h +++ b/CPP/7zip/Compress/BitmDecoder.h @@ -61,6 +61,12 @@ class CDecoder return ((_value >> (8 - _bitPos)) & kMask) >> (kNumValueBits - numBits); } + Z7_FORCE_INLINE + UInt32 GetValue_InHigh32bits() const + { + return this->_value << this->_bitPos; + } + Z7_FORCE_INLINE void MovePos(unsigned numBits) { diff --git a/CPP/7zip/Compress/BranchMisc.cpp b/CPP/7zip/Compress/BranchMisc.cpp index d9fec8c..2098beb 100644 --- a/CPP/7zip/Compress/BranchMisc.cpp +++ b/CPP/7zip/Compress/BranchMisc.cpp @@ -26,8 +26,6 @@ Z7_COM7F_IMF2(UInt32, CCoder::Filter(Byte *data, UInt32 size)) } -namespace NArm64 { - #ifndef Z7_EXTRACT_ONLY Z7_COM7F_IMF(CEncoder::Init()) @@ -38,7 +36,7 @@ Z7_COM7F_IMF(CEncoder::Init()) Z7_COM7F_IMF2(UInt32, CEncoder::Filter(Byte *data, UInt32 size)) { - const UInt32 processed = (UInt32)(size_t)(Z7_BRANCH_CONV_ENC(ARM64)(data, size, _pc) - data); + const UInt32 processed = (UInt32)(size_t)(BraFunc(data, size, _pc) - data); _pc += processed; return processed; } @@ -56,7 +54,7 @@ Z7_COM7F_IMF(CEncoder::SetCoderProperties(const PROPID *propIDs, const PROPVARIA if (prop.vt != VT_UI4) return E_INVALIDARG; pc = prop.ulVal; - if ((pc & 3) != 0) + if (pc & _alignment) return E_INVALIDARG; } } @@ -69,9 +67,9 @@ Z7_COM7F_IMF(CEncoder::WriteCoderProperties(ISequentialOutStream *outStream)) { if (_pc_Init == 0) return S_OK; - Byte buf[4]; - SetUi32(buf, _pc_Init) - return WriteStream(outStream, buf, 4); + UInt32 buf32[1]; + SetUi32(buf32, _pc_Init) + return WriteStream(outStream, buf32, 4); } #endif @@ -85,7 +83,7 @@ Z7_COM7F_IMF(CDecoder::Init()) Z7_COM7F_IMF2(UInt32, CDecoder::Filter(Byte *data, UInt32 size)) { - const UInt32 processed = (UInt32)(size_t)(Z7_BRANCH_CONV_DEC(ARM64)(data, size, _pc) - data); + const UInt32 processed = (UInt32)(size_t)(BraFunc(data, size, _pc) - data); _pc += processed; return processed; } @@ -98,13 +96,11 @@ Z7_COM7F_IMF(CDecoder::SetDecoderProperties2(const Byte *props, UInt32 size)) if (size != 4) return E_NOTIMPL; val = GetUi32(props); - if ((val & 3) != 0) + if (val & _alignment) return E_NOTIMPL; } _pc_Init = val; return S_OK; } -} - }} diff --git a/CPP/7zip/Compress/BranchMisc.h b/CPP/7zip/Compress/BranchMisc.h index a5793f7..2b5cc42 100644 --- a/CPP/7zip/Compress/BranchMisc.h +++ b/CPP/7zip/Compress/BranchMisc.h @@ -21,8 +21,6 @@ Z7_CLASS_IMP_COM_1( CCoder(z7_Func_BranchConv bra): _pc(0), BraFunc(bra) {} }; -namespace NArm64 { - #ifndef Z7_EXTRACT_ONLY Z7_CLASS_IMP_COM_3( @@ -33,8 +31,11 @@ Z7_CLASS_IMP_COM_3( ) UInt32 _pc; UInt32 _pc_Init; + UInt32 _alignment; + z7_Func_BranchConv BraFunc; public: - CEncoder(): _pc(0), _pc_Init(0) {} + CEncoder(z7_Func_BranchConv bra, UInt32 alignment): + _pc(0), _pc_Init(0), _alignment(alignment), BraFunc(bra) {} }; #endif @@ -46,12 +47,13 @@ Z7_CLASS_IMP_COM_2( ) UInt32 _pc; UInt32 _pc_Init; + UInt32 _alignment; + z7_Func_BranchConv BraFunc; public: - CDecoder(): _pc(0), _pc_Init(0) {} + CDecoder(z7_Func_BranchConv bra, UInt32 alignment): + _pc(0), _pc_Init(0), _alignment(alignment), BraFunc(bra) {} }; -} - }} #endif diff --git a/CPP/7zip/Compress/BranchRegister.cpp b/CPP/7zip/Compress/BranchRegister.cpp index 80fbf60..20385ff 100644 --- a/CPP/7zip/Compress/BranchRegister.cpp +++ b/CPP/7zip/Compress/BranchRegister.cpp @@ -15,18 +15,18 @@ namespace NBranch { #else #define GET_CREATE_FUNC(x) x #define CREATE_BRA_E(n) \ - REGISTER_FILTER_CREATE(CreateBra_Encoder_ ## n, CCoder(Z7_BRANCH_CONV_ENC(n))) + REGISTER_FILTER_CREATE(CreateBra_Encoder_ ## n, CCoder(Z7_BRANCH_CONV_ENC_2(n))) #endif #define CREATE_BRA(n) \ - REGISTER_FILTER_CREATE(CreateBra_Decoder_ ## n, CCoder(Z7_BRANCH_CONV_DEC(n))) \ + REGISTER_FILTER_CREATE(CreateBra_Decoder_ ## n, CCoder(Z7_BRANCH_CONV_DEC_2(n))) \ CREATE_BRA_E(n) -CREATE_BRA(PPC) -CREATE_BRA(IA64) -CREATE_BRA(ARM) -CREATE_BRA(ARMT) -CREATE_BRA(SPARC) +CREATE_BRA(BranchConv_PPC) +CREATE_BRA(BranchConv_IA64) +CREATE_BRA(BranchConv_ARM) +CREATE_BRA(BranchConv_ARMT) +CREATE_BRA(BranchConv_SPARC) #define METHOD_ITEM(n, id, name) \ REGISTER_FILTER_ITEM( \ @@ -36,20 +36,23 @@ CREATE_BRA(SPARC) REGISTER_CODECS_VAR { - METHOD_ITEM(PPC, 0x205, "PPC"), - METHOD_ITEM(IA64, 0x401, "IA64"), - METHOD_ITEM(ARM, 0x501, "ARM"), - METHOD_ITEM(ARMT, 0x701, "ARMT"), - METHOD_ITEM(SPARC, 0x805, "SPARC") + METHOD_ITEM(BranchConv_PPC, 0x205, "PPC"), + METHOD_ITEM(BranchConv_IA64, 0x401, "IA64"), + METHOD_ITEM(BranchConv_ARM, 0x501, "ARM"), + METHOD_ITEM(BranchConv_ARMT, 0x701, "ARMT"), + METHOD_ITEM(BranchConv_SPARC, 0x805, "SPARC") }; REGISTER_CODECS(Branch) -namespace NArm64 { -REGISTER_FILTER_E(ARM64, - CDecoder(), - CEncoder(), - 0xa, "ARM64") -} + +#define REGISTER_FILTER_E_BRANCH(id, n, name, alignment) \ + REGISTER_FILTER_E(n, \ + CDecoder(Z7_BRANCH_CONV_DEC(n), alignment), \ + CEncoder(Z7_BRANCH_CONV_ENC(n), alignment), \ + id, name) + +REGISTER_FILTER_E_BRANCH(0xa, ARM64, "ARM64", 3) +REGISTER_FILTER_E_BRANCH(0xb, RISCV, "RISCV", 1) }} diff --git a/CPP/7zip/Compress/DeflateDecoder.cpp b/CPP/7zip/Compress/DeflateDecoder.cpp index 017b694..73895fe 100644 --- a/CPP/7zip/Compress/DeflateDecoder.cpp +++ b/CPP/7zip/Compress/DeflateDecoder.cpp @@ -15,8 +15,8 @@ CCoder::CCoder(bool deflate64Mode): _needFinishInput(false), _needInitInStream(true), _outSizeDefined(false), - _outStartPos(0), - ZlibMode(false) {} + _outStartPos(0) + {} UInt32 CCoder::ReadBits(unsigned numBits) { @@ -34,7 +34,7 @@ bool CCoder::DecodeLevels(Byte *levels, unsigned numSymbols) do { - UInt32 sym = m_LevelDecoder.Decode(&m_InBitStream); + unsigned sym = m_LevelDecoder.Decode(&m_InBitStream); if (sym < kTableDirectLevels) levels[i++] = (Byte)sym; else @@ -110,7 +110,7 @@ bool CCoder::ReadTables(void) else { const unsigned numLitLenLevels = ReadBits(kNumLenCodesFieldSize) + kNumLitLenCodesMin; - _numDistLevels = ReadBits(kNumDistCodesFieldSize) + kNumDistCodesMin; + _numDistLevels = (unsigned)ReadBits(kNumDistCodesFieldSize) + kNumDistCodesMin; const unsigned numLevelCodes = ReadBits(kNumLevelCodesFieldSize) + kNumLevelCodesMin; if (!_deflate64Mode) @@ -130,7 +130,7 @@ bool CCoder::ReadTables(void) if (m_InBitStream.ExtraBitsWereRead()) return false; - RIF(m_LevelDecoder.Build(levelLevels)) + RIF(m_LevelDecoder.Build(levelLevels, false)) // full Byte tmpLevels[kFixedMainTableSize + kFixedDistTableSize]; if (!DecodeLevels(tmpLevels, numLitLenLevels + _numDistLevels)) @@ -182,7 +182,8 @@ HRESULT CCoder::CodeSpec(UInt32 curSize, bool finishInputStream, UInt32 inputPro _needReadTable = true; } - while (_remainLen > 0 && curSize > 0) + // _remainLen >= 0 + while (_remainLen && curSize) { _remainLen--; const Byte b = m_OutWindowStream.GetByte(_rep0); @@ -194,7 +195,7 @@ HRESULT CCoder::CodeSpec(UInt32 curSize, bool finishInputStream, UInt32 inputPro if (inputProgressLimit != 0) inputStart = m_InBitStream.GetProcessedSize(); - while (curSize > 0 || finishInputStream) + while (curSize || finishInputStream) { if (m_InBitStream.ExtraBitsWereRead()) return S_FALSE; @@ -224,20 +225,53 @@ HRESULT CCoder::CodeSpec(UInt32 curSize, bool finishInputStream, UInt32 inputPro return S_FALSE; /* NSIS version contains some bits in bitl bits buffer. So we must read some first bytes via ReadAlignedByte */ - for (; m_StoredBlockSize > 0 && curSize > 0 && m_InBitStream.ThereAreDataInBitsBuffer(); m_StoredBlockSize--, curSize--) + UInt32 num = m_StoredBlockSize; + if (num > curSize) + num = curSize; + m_StoredBlockSize -= num; + curSize -= num; + for (; num && m_InBitStream.ThereAreDataInBitsBuffer(); num--) m_OutWindowStream.PutByte(ReadAlignedByte()); - for (; m_StoredBlockSize > 0 && curSize > 0; m_StoredBlockSize--, curSize--) - m_OutWindowStream.PutByte(m_InBitStream.ReadDirectByte()); + if (num) + { +#if 1 + // fast code + do + { + size_t a; + Byte *buf = m_OutWindowStream.GetOutBuffer(a); + // a != 0 + if (a > num) + a = num; + // a != 0 + a = m_InBitStream.ReadDirectBytesPart(buf, a); + if (a == 0) + return S_FALSE; + m_OutWindowStream.SkipWrittenBytes(a); + num -= (UInt32)a; + } + while (num); +#else + // slow code: + do + m_OutWindowStream.PutByte(m_InBitStream.ReadDirectByte()); + while (--num); +#endif + } _needReadTable = (m_StoredBlockSize == 0); continue; } - while (curSize > 0) + while (curSize) { if (m_InBitStream.ExtraBitsWereRead_Fast()) return S_FALSE; - - UInt32 sym = m_MainDecoder.Decode(&m_InBitStream); + unsigned sym; +#if 0 + sym = m_MainDecoder.Decode(&m_InBitStream); +#else + Z7_HUFF_DECODE_CHECK(sym, &m_MainDecoder, kNumHuffmanBits, kNumTableBits_Main, &m_InBitStream, { return S_FALSE; }) +#endif if (sym < 0x100) { @@ -245,12 +279,15 @@ HRESULT CCoder::CodeSpec(UInt32 curSize, bool finishInputStream, UInt32 inputPro curSize--; continue; } - else if (sym == kSymbolEndOfBlock) + if (sym == kSymbolEndOfBlock) { _needReadTable = true; break; } - else if (sym < kMainTableSize) +#if 0 + if (sym >= kMainTableSize) + return S_FALSE; +#endif { sym -= kSymbolMatch; UInt32 len; @@ -268,22 +305,29 @@ HRESULT CCoder::CodeSpec(UInt32 curSize, bool finishInputStream, UInt32 inputPro } len += kMatchMinLen + m_InBitStream.ReadBits(numBits); } - UInt32 locLen = len; - if (locLen > curSize) - locLen = (UInt32)curSize; + +#if 0 sym = m_DistDecoder.Decode(&m_InBitStream); if (sym >= _numDistLevels) return S_FALSE; +#else + Z7_HUFF_DECODE_CHECK(sym, &m_DistDecoder, kNumHuffmanBits, kNumTableBits_Dist, &m_InBitStream, { return S_FALSE; }) +#endif + +#if 1 sym = kDistStart[sym] + m_InBitStream.ReadBits(kDistDirectBits[sym]); - /* +#else if (sym >= 4) { // sym &= 31; - const unsigned numDirectBits = (unsigned)(((sym >> 1) - 1)); - sym = (2 | (sym & 1)) << numDirectBits; + const unsigned numDirectBits = (sym - 2) >> 1; + sym = (2u | (sym & 1)) << numDirectBits; sym += m_InBitStream.ReadBits(numDirectBits); } - */ +#endif + UInt32 locLen = len; + if (locLen > curSize) + locLen = (UInt32)curSize; if (!m_OutWindowStream.CopyBlock(sym, locLen)) return S_FALSE; curSize -= locLen; @@ -295,8 +339,6 @@ HRESULT CCoder::CodeSpec(UInt32 curSize, bool finishInputStream, UInt32 inputPro break; } } - else - return S_FALSE; } if (finishInputStream && curSize == 0) @@ -354,12 +396,12 @@ HRESULT CCoder::CodeReal(ISequentialOutStream *outStream, ICompressProgressInfo if (curSize >= rem) { curSize = (UInt32)rem; - if (ZlibMode || _needFinishInput) + if (_needFinishInput) finishInputStream = true; + else if (curSize == 0) + break; } } - if (!finishInputStream && curSize == 0) - break; RINOK(CodeSpec(curSize, finishInputStream, progress ? kInputProgressLimit : 0)) @@ -374,13 +416,6 @@ HRESULT CCoder::CodeReal(ISequentialOutStream *outStream, ICompressProgressInfo } } - if (_remainLen == kLenIdFinished && ZlibMode) - { - m_InBitStream.AlignToByte(); - for (unsigned i = 0; i < 4; i++) - ZlibFooter[i] = ReadAlignedByte(); - } - flusher.NeedFlush = false; res = Flush(); if (res == S_OK && _remainLen != kLenIdNeedInit && InputEofError()) @@ -450,6 +485,7 @@ Z7_COM7F_IMF(CCoder::SetInStream(ISequentialInStream *inStream)) Z7_COM7F_IMF(CCoder::ReleaseInStream()) { m_InStreamRef.Release(); + m_InBitStream.ClearStreamPtr(); return S_OK; } @@ -460,10 +496,8 @@ void CCoder::SetOutStreamSizeResume(const UInt64 *outSize) _outSize = 0; if (_outSizeDefined) _outSize = *outSize; - m_OutWindowStream.Init(_keepHistory); _outStartPos = m_OutWindowStream.GetProcessedSize(); - _remainLen = kLenIdNeedInit; } @@ -488,8 +522,6 @@ Z7_COM7F_IMF(CCoder::SetOutStreamSize(const UInt64 *outSize)) Z7_COM7F_IMF(CCoder::Read(void *data, UInt32 size, UInt32 *processedSize)) { - HRESULT res; - if (processedSize) *processedSize = 0; const UInt64 outPos = GetOutProcessedCur(); @@ -501,30 +533,25 @@ Z7_COM7F_IMF(CCoder::Read(void *data, UInt32 size, UInt32 *processedSize)) if (size >= rem) { size = (UInt32)rem; - if (ZlibMode || _needFinishInput) + if (_needFinishInput) finishInputStream = true; } } if (!finishInputStream && size == 0) return S_OK; + HRESULT res; DEFLATE_TRY_BEGIN - m_OutWindowStream.SetMemStream((Byte *)data); - res = CodeSpec(size, finishInputStream); - DEFLATE_TRY_END(res) - { const HRESULT res2 = Flush(); if (res2 != S_OK) res = res2; } - if (processedSize) *processedSize = (UInt32)(GetOutProcessedCur() - outPos); - m_OutWindowStream.SetMemStream(NULL); return res; } diff --git a/CPP/7zip/Compress/DeflateDecoder.h b/CPP/7zip/Compress/DeflateDecoder.h index d31f299..4e0ae61 100644 --- a/CPP/7zip/Compress/DeflateDecoder.h +++ b/CPP/7zip/Compress/DeflateDecoder.h @@ -21,6 +21,9 @@ namespace NDecoder { const int kLenIdFinished = -1; const int kLenIdNeedInit = -2; +const unsigned kNumTableBits_Main = 10; +const unsigned kNumTableBits_Dist = 6; + class CCoder: public ICompressCoder, public ICompressSetFinishMode, @@ -28,9 +31,9 @@ class CCoder: public ICompressReadUnusedFromInBuf, public ICompressSetInStream, public ICompressSetOutStreamSize, - #ifndef Z7_NO_READ_FROM_CODER +#ifndef Z7_NO_READ_FROM_CODER public ISequentialInStream, - #endif +#endif public CMyUnknownImp { Z7_COM_QI_BEGIN2(ICompressCoder) @@ -39,34 +42,33 @@ class CCoder: Z7_COM_QI_ENTRY(ICompressReadUnusedFromInBuf) Z7_COM_QI_ENTRY(ICompressSetInStream) Z7_COM_QI_ENTRY(ICompressSetOutStreamSize) - #ifndef Z7_NO_READ_FROM_CODER +#ifndef Z7_NO_READ_FROM_CODER Z7_COM_QI_ENTRY(ISequentialInStream) - #endif +#endif Z7_COM_QI_END Z7_COM_ADDREF_RELEASE Z7_IFACE_COM7_IMP(ICompressCoder) Z7_IFACE_COM7_IMP(ICompressSetFinishMode) Z7_IFACE_COM7_IMP(ICompressGetInStreamProcessedSize) - Z7_IFACE_COM7_IMP(ICompressReadUnusedFromInBuf) public: + Z7_IFACE_COM7_IMP(ICompressReadUnusedFromInBuf) Z7_IFACE_COM7_IMP(ICompressSetInStream) private: Z7_IFACE_COM7_IMP(ICompressSetOutStreamSize) - #ifndef Z7_NO_READ_FROM_CODER +#ifndef Z7_NO_READ_FROM_CODER Z7_IFACE_COM7_IMP(ISequentialInStream) - #endif +#endif CLzOutWindow m_OutWindowStream; - CMyComPtr m_InStreamRef; NBitl::CDecoder m_InBitStream; - NCompress::NHuffman::CDecoder m_MainDecoder; - NCompress::NHuffman::CDecoder m_DistDecoder; + NCompress::NHuffman::CDecoder m_MainDecoder; + NCompress::NHuffman::CDecoder256 m_DistDecoder; NCompress::NHuffman::CDecoder7b m_LevelDecoder; UInt32 m_StoredBlockSize; - UInt32 _numDistLevels; + unsigned _numDistLevels; bool m_FinalBlock; bool m_StoredMode; @@ -81,6 +83,7 @@ class CCoder: UInt32 _rep0; bool _outSizeDefined; + CMyComPtr m_InStreamRef; UInt64 _outSize; UInt64 _outStartPos; @@ -109,8 +112,6 @@ class CCoder: HRESULT CodeSpec(UInt32 curSize, bool finishInputStream, UInt32 inputProgressLimit = 0); public: - bool ZlibMode; - Byte ZlibFooter[4]; CCoder(bool deflate64Mode); virtual ~CCoder() {} @@ -133,7 +134,7 @@ class CCoder: Byte ReadAlignedByte(); UInt32 ReadAligned_UInt16() // aligned for Byte range { - UInt32 v = m_InBitStream.ReadAlignedByte(); + const UInt32 v = m_InBitStream.ReadAlignedByte(); return v | ((UInt32)m_InBitStream.ReadAlignedByte() << 8); } bool InputEofError() const { return m_InBitStream.ExtraBitsWereRead(); } diff --git a/CPP/7zip/Compress/DeflateEncoder.cpp b/CPP/7zip/Compress/DeflateEncoder.cpp index e277ad6..87b4f83 100644 --- a/CPP/7zip/Compress/DeflateEncoder.cpp +++ b/CPP/7zip/Compress/DeflateEncoder.cpp @@ -43,9 +43,11 @@ static const Byte kNoLiteralStatPrice = 11; static const Byte kNoLenStatPrice = 11; static const Byte kNoPosStatPrice = 6; +MY_ALIGN(64) static Byte g_LenSlots[kNumLenSymbolsMax]; #define kNumLogBits 9 // do not change it +MY_ALIGN(64) static Byte g_FastPos[1 << kNumLogBits]; class CFastPosInit @@ -57,7 +59,7 @@ class CFastPosInit for (i = 0; i < kNumLenSlots; i++) { unsigned c = kLenStart32[i]; - unsigned j = 1 << kLenDirectBits32[i]; + const unsigned j = 1u << kLenDirectBits32[i]; for (unsigned k = 0; k < j; k++, c++) g_LenSlots[c] = (Byte)i; } @@ -66,8 +68,8 @@ class CFastPosInit unsigned c = 0; for (Byte slotFast = 0; slotFast < kFastSlots; slotFast++) { - UInt32 k = (1 << kDistDirectBits[slotFast]); - for (UInt32 j = 0; j < k; j++, c++) + const unsigned k = 1u << kDistDirectBits[slotFast]; + for (unsigned j = 0; j < k; j++, c++) g_FastPos[c] = slotFast; } } diff --git a/CPP/7zip/Compress/DeltaFilter.cpp b/CPP/7zip/Compress/DeltaFilter.cpp index 35b77ba..c234b7a 100644 --- a/CPP/7zip/Compress/DeltaFilter.cpp +++ b/CPP/7zip/Compress/DeltaFilter.cpp @@ -52,7 +52,7 @@ Z7_COM7F_IMF2(UInt32, CEncoder::Filter(Byte *data, UInt32 size)) Z7_COM7F_IMF(CEncoder::SetCoderProperties(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps)) { - UInt32 delta = _delta; + unsigned delta = _delta; for (UInt32 i = 0; i < numProps; i++) { const PROPVARIANT &prop = props[i]; @@ -64,9 +64,9 @@ Z7_COM7F_IMF(CEncoder::SetCoderProperties(const PROPID *propIDs, const PROPVARIA switch (propID) { case NCoderPropID::kDefaultProp: - delta = (UInt32)prop.ulVal; - if (delta < 1 || delta > 256) + if (prop.ulVal < 1 || prop.ulVal > 256) return E_INVALIDARG; + delta = prop.ulVal; break; case NCoderPropID::kNumThreads: break; case NCoderPropID::kLevel: break; diff --git a/CPP/7zip/Compress/HuffmanDecoder.h b/CPP/7zip/Compress/HuffmanDecoder.h index 4b8169f..318190d 100644 --- a/CPP/7zip/Compress/HuffmanDecoder.h +++ b/CPP/7zip/Compress/HuffmanDecoder.h @@ -3,178 +3,401 @@ #ifndef ZIP7_INC_COMPRESS_HUFFMAN_DECODER_H #define ZIP7_INC_COMPRESS_HUFFMAN_DECODER_H +#include "../../../C/CpuArch.h" #include "../../Common/MyTypes.h" namespace NCompress { namespace NHuffman { -const unsigned kNumPairLenBits = 4; -const unsigned kPairLenMask = (1 << kNumPairLenBits) - 1; +// const unsigned kNumTableBits_Default = 9; + +#if 0 || 0 && defined(MY_CPU_64BIT) +// for debug or optimization: +// 64-BIT limit array can be faster for some compilers. +// for debug or optimization: +#define Z7_HUFF_USE_64BIT_LIMIT +#else +// sizet value variable allows to eliminate some move operation in some compilers. +// for debug or optimization: +// #define Z7_HUFF_USE_SIZET_VALUE +#endif + +// v0 must normalized to (32 bits) : (v0 < ((UInt64)1 << 32)) + +#ifdef Z7_HUFF_USE_64BIT_LIMIT + typedef UInt64 CLimitInt; + typedef UInt64 CValueInt; + // all _limits[*] are normalized and limited by ((UInt64)1 << 32). + // we don't use (v1) in this branch + #define Z7_HUFF_NUM_LIMIT_BITS(kNumBitsMax) 32 + #define Z7_HUFF_TABLE_COMPARE(huf, kNumTableBits, v0, v1) \ + ((NCompress::NHuffman::CLimitInt)v0 >= (huf)->_limits[0]) + #define Z7_HUFF_GET_VAL_FOR_LIMITS(v0, v1, kNumBitsMax, kNumTableBits) (v0) + #define Z7_HUFF_GET_VAL_FOR_TABLE( v0, v1, kNumBitsMax, kNumTableBits) ((v0) >> (32 - kNumTableBits)) + #define Z7_HUFF_PRECALC_V1(kNumTableBits, v0, v1) +#else + typedef UInt32 CLimitInt; + typedef + #ifdef Z7_HUFF_USE_SIZET_VALUE + size_t + #else + UInt32 + #endif + CValueInt; + // v1 must be precalculated from v0 in this branch + // _limits[0] and (v1) are normalized and limited by (1 << kNumTableBits). + // _limits[non_0] are normalized and limited by (1 << kNumBitsMax). + #define Z7_HUFF_NUM_LIMIT_BITS(kNumBitsMax) (kNumBitsMax) + #define Z7_HUFF_TABLE_COMPARE(huf, kNumTableBits, v0, v1) \ + ((NCompress::NHuffman::CLimitInt)v1 >= (huf)->_limits[0]) + #define Z7_HUFF_GET_VAL_FOR_LIMITS(v0, v1, kNumBitsMax, kNumTableBits) ((v0) >> (32 - kNumBitsMax)) + #define Z7_HUFF_GET_VAL_FOR_TABLE( v0, v1, kNumBitsMax, kNumTableBits) (v1) + #define Z7_HUFF_PRECALC_V1(kNumTableBits, v0, v1) const UInt32 v1 = ((v0) >> (32 - kNumTableBits)); +#endif -template -class CDecoder + +enum enum_BuildMode { -public: - UInt32 _limits[kNumBitsMax + 2]; - UInt32 _poses[kNumBitsMax + 1]; - UInt16 _lens[1 << kNumTableBits]; - UInt16 _symbols[m_NumSymbols]; + k_BuildMode_Partial = 0, + k_BuildMode_Full = 1, + k_BuildMode_Full_or_Empty = 2 +}; + - bool Build(const Byte *lens) throw() +template +struct CDecoderBase +{ + CLimitInt _limits[kNumBitsMax + 2 - kNumTableBits]; + UInt32 _poses[kNumBitsMax - kNumTableBits]; // unsigned +union +{ + // if defined(MY_CPU_64BIT), we need 64-bit alignment for _symbols. + // if !defined(MY_CPU_64BIT), we need 32-bit alignment for _symbols + // but we provide alignment for _lens. + // _symbols also will be aligned, if _lens are aligned + #if defined(MY_CPU_64BIT) + UInt64 + #else + UInt32 + #endif + _pad_align[m_NumSymbols < (1u << sizeof(symType) * 8) ? 1 : -1]; + /* if symType is Byte, we use 16-bytes padding to avoid cache + bank conflict between _lens and _symbols: */ + Byte _lens[(1 << kNumTableBits) + (sizeof(symType) == 1 ? 16 : 0)]; +} _u; + symType _symbols[(1 << kNumTableBits) + m_NumSymbols - (kNumTableBits + 1)]; + + /* + Z7_FORCE_INLINE + bool IsFull() const { - UInt32 counts[kNumBitsMax + 1]; - - unsigned i; + return _limits[kNumBitsMax - kNumTableBits] == + (CLimitInt)1u << Z7_HUFF_NUM_LIMIT_BITS(kNumBitsMax); + } + Z7_FORCE_INLINE + bool IsEmpty() const + { + return _limits[kNumBitsMax - kNumTableBits] == 0; + } + Z7_FORCE_INLINE + bool Is_Full_or_Empty() const + { + return 0 == (_limits[kNumBitsMax - kNumTableBits] & + ~((CLimitInt)1 << Z7_HUFF_NUM_LIMIT_BITS(kNumBitsMax))); + } + */ + + Z7_FORCE_INLINE + bool Build(const Byte *lens, enum_BuildMode buidMode = k_BuildMode_Partial) throw() + { + unsigned counts[kNumBitsMax + 1]; + size_t i; for (i = 0; i <= kNumBitsMax; i++) counts[i] = 0; + for (i = 0; i < m_NumSymbols; i++) + counts[lens[i]]++; - UInt32 sym; - - for (sym = 0; sym < m_NumSymbols; sym++) - counts[lens[sym]]++; - - const UInt32 kMaxValue = (UInt32)1 << kNumBitsMax; - - _limits[0] = 0; - - UInt32 startPos = 0; UInt32 sum = 0; - - for (i = 1; i <= kNumBitsMax; i++) + for (i = 1; i <= kNumTableBits; i++) { - const UInt32 cnt = counts[i]; - startPos += cnt << (kNumBitsMax - i); - if (startPos > kMaxValue) - return false; - _limits[i] = startPos; + sum <<= 1; + sum += counts[i]; + } + + CLimitInt startPos = (CLimitInt)sum; + _limits[0] = + #ifdef Z7_HUFF_USE_64BIT_LIMIT + startPos << (Z7_HUFF_NUM_LIMIT_BITS(kNumBitsMax) - kNumTableBits); + #else + startPos; + #endif + + for (i = kNumTableBits + 1; i <= kNumBitsMax; i++) + { + startPos <<= 1; + _poses[i - (kNumTableBits + 1)] = (UInt32)(startPos - sum); + const unsigned cnt = counts[i]; counts[i] = sum; - _poses[i] = sum; sum += cnt; + startPos += cnt; + _limits[i - kNumTableBits] = startPos << (Z7_HUFF_NUM_LIMIT_BITS(kNumBitsMax) - i); } - counts[0] = sum; - _poses[0] = sum; - _limits[kNumBitsMax + 1] = kMaxValue; + _limits[kNumBitsMax + 1 - kNumTableBits] = + (CLimitInt)1 << Z7_HUFF_NUM_LIMIT_BITS(kNumBitsMax); + + if (buidMode == k_BuildMode_Partial) + { + if (startPos > (1u << kNumBitsMax)) return false; + } + else + { + if (buidMode != k_BuildMode_Full && startPos == 0) return true; + if (startPos != (1u << kNumBitsMax)) return false; + } + size_t sum2 = 0; + for (i = 1; i <= kNumTableBits; i++) + { + const unsigned cnt = counts[i] << (kNumTableBits - i); + counts[i] = (unsigned)sum2 >> (kNumTableBits - i); + memset(_u._lens + sum2, (int)i, cnt); + sum2 += cnt; + } - for (sym = 0; sym < m_NumSymbols; sym++) +#ifdef MY_CPU_64BIT + symType4 + // UInt64 // for symType = UInt16 + // UInt32 // for symType = Byte +#else + UInt32 +#endif + v = 0; + for (i = 0; i < m_NumSymbols; i++, + v += + 1 + + ( (UInt32)1 << (sizeof(symType) * 8 * 1)) + // 0x00010001 // for symType = UInt16 + // 0x00000101 // for symType = Byte +#ifdef MY_CPU_64BIT + + ((symType4)1 << (sizeof(symType) * 8 * 2)) + + ((symType4)1 << (sizeof(symType) * 8 * 3)) + // 0x0001000100010001 // for symType = UInt16 + // 0x0000000001010101 // for symType = Byte +#endif + ) { - unsigned len = lens[sym]; + const unsigned len = lens[i]; if (len == 0) continue; - - unsigned offset = counts[len]++; - _symbols[offset] = (UInt16)sym; - - if (len <= kNumTableBits) + const size_t offset = counts[len]; + counts[len] = (unsigned)offset + 1; + if (len >= kNumTableBits) + _symbols[offset] = (symType)v; + else { - offset -= _poses[len]; - UInt32 num = (UInt32)1 << (kNumTableBits - len); - UInt16 val = (UInt16)((sym << kNumPairLenBits) | len); - UInt16 *dest = _lens + (_limits[(size_t)len - 1] >> (kNumBitsMax - kNumTableBits)) + (offset << (kNumTableBits - len)); - for (UInt32 k = 0; k < num; k++) - dest[k] = val; + Byte *s2 = (Byte *)(void *)_symbols + (offset << + (kNumTableBits + sizeof(symType) / 2 - len)); + Byte *lim = s2 + ((size_t)1 << + (kNumTableBits + sizeof(symType) / 2 - len)); + if (len >= kNumTableBits - 2) + { + *(symType2 *)(void *)(s2 ) = (symType2)v; + *(symType2 *)(void *)(lim - sizeof(symType) * 2) = (symType2)v; + } + else + { +#ifdef MY_CPU_64BIT + symType4 *s = (symType4 *)(void *)s2; + do + { + s[0] = v; s[1] = v; s += 2; + } + while (s != (const symType4 *)(const void *)lim); +#else + symType2 *s = (symType2 *)(void *)s2; + do + { + s[0] = (symType2)v; s[1] = (symType2)v; s += 2; + s[0] = (symType2)v; s[1] = (symType2)v; s += 2; + } + while (s != (const symType2 *)(const void *)lim); +#endif + } } } - return true; } - bool BuildFull(const Byte *lens, UInt32 numSymbols = m_NumSymbols) throw() - { - UInt32 counts[kNumBitsMax + 1]; - - unsigned i; - for (i = 0; i <= kNumBitsMax; i++) - counts[i] = 0; - - UInt32 sym; - - for (sym = 0; sym < numSymbols; sym++) - counts[lens[sym]]++; - - const UInt32 kMaxValue = (UInt32)1 << kNumBitsMax; +#define Z7_HUFF_DECODE_ERROR_SYM_CHECK_YES(_numBits_, kNumBitsMax, error_op) if (_numBits_ > kNumBitsMax) { error_op } +#define Z7_HUFF_DECODE_ERROR_SYM_CHECK_NO( _numBits_, kNumBitsMax, error_op) - _limits[0] = 0; - - UInt32 startPos = 0; - UInt32 sum = 0; - - for (i = 1; i <= kNumBitsMax; i++) - { - const UInt32 cnt = counts[i]; - startPos += cnt << (kNumBitsMax - i); - if (startPos > kMaxValue) - return false; - _limits[i] = startPos; - counts[i] = sum; - _poses[i] = sum; - sum += cnt; - } + +#define Z7_HUFF_DECODE_BASE_TREE_BRANCH(sym, huf, kNumBitsMax, kNumTableBits, \ + v0, v1, \ + get_val_for_limits, \ + check_op, error_op, _numBits_) \ +{ \ + const NHuffman::CValueInt _val = get_val_for_limits(v0, v1, kNumBitsMax, kNumTableBits); \ + _numBits_ = kNumTableBits + 1; \ + if ((NCompress::NHuffman::CLimitInt)_val >= (huf)->_limits[1]) \ + do { _numBits_++; } \ + while ((NCompress::NHuffman::CLimitInt)_val >= (huf)->_limits[_numBits_ - kNumTableBits]); \ + check_op(_numBits_, kNumBitsMax, error_op) \ + sym = (huf)->_symbols[(/* (UInt32) */ (_val >> ((Z7_HUFF_NUM_LIMIT_BITS(kNumBitsMax) - (unsigned)_numBits_)))) \ + - (huf)->_poses[_numBits_ - (kNumTableBits + 1)]]; \ +} + +/* + Z7_HUFF_DECODE_BASE_TREE_BRANCH(sym, huf, kNumBitsMax, kNumTableBits, \ + v0, v1, \ + get_val_for_limits, \ + check_op, error_op, _numBits_) \ + +*/ + +#define Z7_HUFF_DECODE_BASE(sym, huf, kNumBitsMax, kNumTableBits, \ + v0, v1, \ + get_val_for_table, get_val_for_limits, \ + check_op, error_op, move_pos_op, after_op, bs) \ +{ \ + if (Z7_HUFF_TABLE_COMPARE(huf, kNumTableBits, v0, v1)) \ + { \ + const NHuffman::CValueInt _val = get_val_for_limits(v0, v1, kNumBitsMax, kNumTableBits); \ + size_t _numBits_ = kNumTableBits + 1; \ + if ((NCompress::NHuffman::CLimitInt)_val >= (huf)->_limits[1]) \ + do { _numBits_++; } \ + while ((NCompress::NHuffman::CLimitInt)_val >= (huf)->_limits[_numBits_ - kNumTableBits]); \ + check_op(_numBits_, kNumBitsMax, error_op) \ + sym = (huf)->_symbols[(/* (UInt32) */ (_val >> ((Z7_HUFF_NUM_LIMIT_BITS(kNumBitsMax) - (unsigned)_numBits_)))) \ + - (huf)->_poses[_numBits_ - (kNumTableBits + 1)]]; \ + move_pos_op(bs, _numBits_); \ + } \ + else \ + { \ + const size_t _val = get_val_for_table(v0, v1, kNumBitsMax, kNumTableBits); \ + const size_t _numBits_ = (huf)->_u._lens[_val]; \ + sym = (huf)->_symbols[_val]; \ + move_pos_op(bs, _numBits_); \ + } \ + after_op \ +} + +#define Z7_HUFF_DECODE_10(sym, huf, kNumBitsMax, kNumTableBits, \ + v0, v1, \ + check_op, error_op, move_pos_op, after_op, bs) \ + Z7_HUFF_DECODE_BASE(sym, huf, kNumBitsMax, kNumTableBits, \ + v0, v1, \ + Z7_HUFF_GET_VAL_FOR_TABLE, \ + Z7_HUFF_GET_VAL_FOR_LIMITS, \ + check_op, error_op, move_pos_op, after_op, bs) \ + + +#define Z7_HUFF_DECODE_VAL_IN_HIGH32(sym, huf, kNumBitsMax, kNumTableBits, \ + v0, \ + check_op, error_op, move_pos_op, after_op, bs) \ +{ \ + Z7_HUFF_PRECALC_V1(kNumTableBits, v0, _v1_temp) \ + Z7_HUFF_DECODE_10(sym, huf, kNumBitsMax, kNumTableBits, \ + v0, _v1_temp, \ + check_op, error_op, move_pos_op, after_op, bs) \ +} + +#if 0 || defined(Z7_HUFF_USE_64BIT_LIMIT) +// this branch uses bitStream->GetValue_InHigh32bits(). +#define Z7_HUFF_DECODE_0(sym, huf, kNumBitsMax, kNumTableBits, bitStream, \ + check_op, error_op, move_pos_op) \ +{ \ + const UInt32 v0 = (bitStream)->GetValue_InHigh32bits(); \ + Z7_HUFF_PRECALC_V1(kNumTableBits, v0, v1); \ + Z7_HUFF_DECODE_BASE(sym, huf, kNumBitsMax, kNumTableBits, \ + v0, v1, \ + Z7_HUFF_GET_VAL_FOR_TABLE, \ + Z7_HUFF_GET_VAL_FOR_LIMITS, \ + check_op, error_op, move_pos_op, {}, bitStream) \ +} +#else +/* +this branch uses bitStream->GetValue(). +So we use SIMPLE versions for v0, v1 calculation: + v0 is normalized for kNumBitsMax + v1 is normalized for kNumTableBits +*/ +#define Z7_HUFF_GET_VAL_FOR_LIMITS_SIMPLE(v0, v1, kNumBitsMax, kNumTableBits) v0 +#define Z7_HUFF_GET_VAL_FOR_TABLE_SIMPLE( v0, v1, kNumBitsMax, kNumTableBits) v1 +#define Z7_HUFF_DECODE_0(sym, huf, kNumBitsMax, kNumTableBits, bitStream, check_op, error_op, move_pos_op) \ +{ \ + const UInt32 v0 = (bitStream)->GetValue(kNumBitsMax); \ + const UInt32 v1 = v0 >> (kNumBitsMax - kNumTableBits); \ + Z7_HUFF_DECODE_BASE(sym, huf, kNumBitsMax, kNumTableBits, \ + v0, v1, \ + Z7_HUFF_GET_VAL_FOR_TABLE_SIMPLE, \ + Z7_HUFF_GET_VAL_FOR_LIMITS_SIMPLE, \ + check_op, error_op, move_pos_op, {}, bitStream) \ +} +#endif - counts[0] = sum; - _poses[0] = sum; - _limits[kNumBitsMax + 1] = kMaxValue; - - for (sym = 0; sym < numSymbols; sym++) - { - unsigned len = lens[sym]; - if (len == 0) - continue; +#define Z7_HUFF_bitStream_MovePos(bitStream, numBits) (bitStream)->MovePos((unsigned)(numBits)) - unsigned offset = counts[len]++; - _symbols[offset] = (UInt16)sym; +#define Z7_HUFF_DECODE_1(sym, huf, kNumBitsMax, kNumTableBits, bitStream, check_op, error_op) \ + Z7_HUFF_DECODE_0(sym, huf, kNumBitsMax, kNumTableBits, bitStream, check_op, error_op, \ + Z7_HUFF_bitStream_MovePos) - if (len <= kNumTableBits) - { - offset -= _poses[len]; - UInt32 num = (UInt32)1 << (kNumTableBits - len); - UInt16 val = (UInt16)((sym << kNumPairLenBits) | len); - UInt16 *dest = _lens + (_limits[(size_t)len - 1] >> (kNumBitsMax - kNumTableBits)) + (offset << (kNumTableBits - len)); - for (UInt32 k = 0; k < num; k++) - dest[k] = val; - } - } - - return startPos == kMaxValue; - } +// MovePosCheck + +#define Z7_HUFF_DECODE_2(sym, huf, kNumBitsMax, kNumTableBits, bitStream, check_op, error_op) \ + Z7_HUFF_DECODE_0(sym, huf, kNumBitsMax, kNumTableBits, bitStream, check_op, error_op, \ + Z7_HUFF_bitStream_MovePos) + +// MovePosCheck + +#define Z7_HUFF_DECODE_CHECK(sym, huf, kNumBitsMax, kNumTableBits, bitStream, error_op) \ + Z7_HUFF_DECODE_1( sym, huf, kNumBitsMax, kNumTableBits, bitStream, \ + Z7_HUFF_DECODE_ERROR_SYM_CHECK_YES, error_op) - template Z7_FORCE_INLINE - UInt32 Decode(TBitDecoder *bitStream) const + bool Decode2(TBitDecoder *bitStream, unsigned &sym) const { - UInt32 val = bitStream->GetValue(kNumBitsMax); - - if (val < _limits[kNumTableBits]) - { - UInt32 pair = _lens[val >> (kNumBitsMax - kNumTableBits)]; - bitStream->MovePos((unsigned)(pair & kPairLenMask)); - return pair >> kNumPairLenBits; - } + Z7_HUFF_DECODE_CHECK(sym, this, kNumBitsMax, kNumTableBits, bitStream, + { return false; } + ) + return true; + } - unsigned numBits; - for (numBits = kNumTableBits + 1; val >= _limits[numBits]; numBits++); - - if (numBits > kNumBitsMax) - return 0xFFFFFFFF; + template + Z7_FORCE_INLINE + bool Decode_SymCheck_MovePosCheck(TBitDecoder *bitStream, unsigned &sym) const + { + Z7_HUFF_DECODE_0(sym, this, kNumBitsMax, kNumTableBits, bitStream, + Z7_HUFF_DECODE_ERROR_SYM_CHECK_YES, + { return false; }, + { return (bitStream)->MovePosCheck; } + ) + } - bitStream->MovePos(numBits); - UInt32 index = _poses[numBits] + ((val - _limits[(size_t)numBits - 1]) >> (kNumBitsMax - numBits)); - return _symbols[index]; + template + Z7_FORCE_INLINE + unsigned Decode(TBitDecoder *bitStream) const + { + unsigned sym; + Z7_HUFF_DECODE_CHECK(sym, this, kNumBitsMax, kNumTableBits, bitStream, + { return (unsigned)(int)(Int32)0xffffffff; } + ) + return sym; } template Z7_FORCE_INLINE - UInt32 DecodeFull(TBitDecoder *bitStream) const + unsigned DecodeFull(TBitDecoder *bitStream) const { - UInt32 val = bitStream->GetValue(kNumBitsMax); - + /* + const UInt32 val = bitStream->GetValue(kNumBitsMax); if (val < _limits[kNumTableBits]) { - UInt32 pair = _lens[val >> (kNumBitsMax - kNumTableBits)]; - bitStream->MovePos((unsigned)(pair & kPairLenMask)); + const unsigned pair = _u._lens[(size_t)(val >> (kNumBitsMax - kNumTableBits))]; + bitStream->MovePos(pair & kPairLenMask); return pair >> kNumPairLenBits; } @@ -182,49 +405,55 @@ class CDecoder for (numBits = kNumTableBits + 1; val >= _limits[numBits]; numBits++); bitStream->MovePos(numBits); - UInt32 index = _poses[numBits] + ((val - _limits[(size_t)numBits - 1]) >> (kNumBitsMax - numBits)); - return _symbols[index]; + return _symbols[_poses[numBits] + (unsigned) + ((val - _limits[(size_t)numBits - 1]) >> (kNumBitsMax - numBits))]; + */ + unsigned sym; + Z7_HUFF_DECODE_2(sym, this, kNumBitsMax, kNumTableBits, bitStream, + Z7_HUFF_DECODE_ERROR_SYM_CHECK_NO, {} + ) + return sym; } }; +template +struct CDecoder: public CDecoderBase + {}; -template +template +struct CDecoder256: public CDecoderBase + {}; + + +template class CDecoder7b { - Byte _lens[1 << 7]; public: + Byte _lens[1 << 7]; - bool Build(const Byte *lens) throw() + bool Build(const Byte *lens, bool full) throw() { const unsigned kNumBitsMax = 7; - UInt32 counts[kNumBitsMax + 1]; - UInt32 _poses[kNumBitsMax + 1]; - UInt32 _limits[kNumBitsMax + 1]; - + unsigned counts[kNumBitsMax + 1]; + unsigned _poses[kNumBitsMax + 1]; + unsigned _limits[kNumBitsMax + 1]; unsigned i; for (i = 0; i <= kNumBitsMax; i++) counts[i] = 0; + for (i = 0; i < numSymbols; i++) + counts[lens[i]]++; - UInt32 sym; - - for (sym = 0; sym < m_NumSymbols; sym++) - counts[lens[sym]]++; - - const UInt32 kMaxValue = (UInt32)1 << kNumBitsMax; - _limits[0] = 0; - - UInt32 startPos = 0; - UInt32 sum = 0; + const unsigned kMaxValue = 1u << kNumBitsMax; + unsigned startPos = 0; + unsigned sum = 0; for (i = 1; i <= kNumBitsMax; i++) { - const UInt32 cnt = counts[i]; + const unsigned cnt = counts[i]; startPos += cnt << (kNumBitsMax - i); - if (startPos > kMaxValue) - return false; _limits[i] = startPos; counts[i] = sum; _poses[i] = sum; @@ -234,41 +463,61 @@ class CDecoder7b counts[0] = sum; _poses[0] = sum; - for (sym = 0; sym < m_NumSymbols; sym++) + if (full) { - unsigned len = lens[sym]; - if (len == 0) - continue; + if (startPos != kMaxValue) + return false; + } + else + { + if (startPos > kMaxValue) + return false; + } - unsigned offset = counts[len]++; + for (i = 0; i < numSymbols; i++) + { + const unsigned len = lens[i]; + if (len == 0) + continue; + const unsigned offset = counts[len]++; { - offset -= _poses[len]; - UInt32 num = (UInt32)1 << (kNumBitsMax - len); - Byte val = (Byte)((sym << 3) | len); - Byte *dest = _lens + (_limits[(size_t)len - 1]) + (offset << (kNumBitsMax - len)); - for (UInt32 k = 0; k < num; k++) - dest[k] = val; + Byte *dest = _lens + _limits[(size_t)len - 1] + + ((offset - _poses[len]) << (kNumBitsMax - len)); + const unsigned num = (unsigned)1 << (kNumBitsMax - len); + const unsigned val = (i << 3) + len; + for (unsigned k = 0; k < num; k++) + dest[k] = (Byte)val; } } + if (!full) { - UInt32 limit = _limits[kNumBitsMax]; - UInt32 num = ((UInt32)1 << kNumBitsMax) - limit; + const unsigned limit = _limits[kNumBitsMax]; + const unsigned num = ((unsigned)1 << kNumBitsMax) - limit; Byte *dest = _lens + limit; - for (UInt32 k = 0; k < num; k++) - dest[k] = (Byte)(0x1F << 3); + for (unsigned k = 0; k < num; k++) + dest[k] = (Byte) + // (0x1f << 3); + ((0x1f << 3) + 0x7); } return true; } +#define Z7_HUFF_DECODER_7B_DECODE(dest, huf, get_val, move_pos, bs) \ + { \ + const unsigned pair = huf->_lens[(size_t)get_val(7)]; \ + const unsigned numBits = pair & 0x7; \ + move_pos(bs, numBits); \ + dest = pair >> 3; \ + } + template - UInt32 Decode(TBitDecoder *bitStream) const + unsigned Decode(TBitDecoder *bitStream) const { - UInt32 val = bitStream->GetValue(7); - UInt32 pair = _lens[val]; - bitStream->MovePos((unsigned)(pair & 0x7)); + const unsigned pair = _lens[(size_t)bitStream->GetValue(7)]; + bitStream->MovePos(pair & 0x7); return pair >> 3; } }; diff --git a/CPP/7zip/Compress/ImplodeDecoder.cpp b/CPP/7zip/Compress/ImplodeDecoder.cpp index 12f2dce..bcf5d4f 100644 --- a/CPP/7zip/Compress/ImplodeDecoder.cpp +++ b/CPP/7zip/Compress/ImplodeDecoder.cpp @@ -13,26 +13,21 @@ namespace NDecoder { bool CHuffmanDecoder::Build(const Byte *lens, unsigned numSymbols) throw() { unsigned counts[kNumHuffmanBits + 1]; - unsigned i; for (i = 0; i <= kNumHuffmanBits; i++) counts[i] = 0; - - unsigned sym; - for (sym = 0; sym < numSymbols; sym++) - counts[lens[sym]]++; + for (i = 0; i < numSymbols; i++) + counts[lens[i]]++; const UInt32 kMaxValue = (UInt32)1 << kNumHuffmanBits; - // _limits[0] = kMaxValue; - UInt32 startPos = kMaxValue; - UInt32 sum = 0; + unsigned sum = 0; for (i = 1; i <= kNumHuffmanBits; i++) { - const UInt32 cnt = counts[i]; - const UInt32 range = cnt << (kNumHuffmanBits - i); + const unsigned cnt = counts[i]; + const UInt32 range = (UInt32)cnt << (kNumHuffmanBits - i); if (startPos < range) return false; startPos -= range; @@ -41,29 +36,26 @@ bool CHuffmanDecoder::Build(const Byte *lens, unsigned numSymbols) throw() sum += cnt; counts[i] = sum; } - // counts[0] += sum; - if (startPos != 0) return false; - - for (sym = 0; sym < numSymbols; sym++) + for (i = 0; i < numSymbols; i++) { - const unsigned len = lens[sym]; + const unsigned len = lens[i]; if (len != 0) - _symbols[--counts[len]] = (Byte)sym; + _symbols[--counts[len]] = (Byte)i; } - return true; } -UInt32 CHuffmanDecoder::Decode(CInBit *inStream) const throw() +unsigned CHuffmanDecoder::Decode(CInBit *inStream) const throw() { const UInt32 val = inStream->GetValue(kNumHuffmanBits); - unsigned numBits; + size_t numBits; for (numBits = 1; val < _limits[numBits]; numBits++); - const UInt32 sym = _symbols[_poses[numBits] + ((val - _limits[numBits]) >> (kNumHuffmanBits - numBits))]; + const unsigned sym = _symbols[_poses[numBits] + + (unsigned)((val - _limits[numBits]) >> (kNumHuffmanBits - numBits))]; inStream->MovePos(numBits); return sym; } @@ -96,12 +88,12 @@ bool CCoder::BuildHuff(CHuffmanDecoder &decoder, unsigned numSymbols) do { const unsigned b = (unsigned)_inBitStream.ReadAlignedByte(); - const Byte level = (Byte)((b & 0xF) + 1); + const unsigned level = (b & 0xF) + 1; const unsigned rep = ((unsigned)b >> 4) + 1; if (index + rep > numSymbols) return false; for (unsigned j = 0; j < rep; j++) - levels[index++] = level; + levels[index++] = (Byte)level; } while (--numRecords); @@ -146,11 +138,11 @@ HRESULT CCoder::CodeReal(ISequentialInStream *inStream, ISequentialOutStream *ou while (pos < unPackSize) { - if (progress && (pos - prevProgress) >= (1 << 18)) + if (pos - prevProgress >= (1u << 18) && progress) { + prevProgress = pos; const UInt64 packSize = _inBitStream.GetProcessedSize(); RINOK(progress->SetRatioInfo(&packSize, &pos)) - prevProgress = pos; } if (_inBitStream.ReadBits(1) != 0) @@ -158,7 +150,7 @@ HRESULT CCoder::CodeReal(ISequentialInStream *inStream, ISequentialOutStream *ou Byte b; if (literalsOn) { - const UInt32 sym = _litDecoder.Decode(&_inBitStream); + const unsigned sym = _litDecoder.Decode(&_inBitStream); // if (sym >= kLitTableSize) break; b = (Byte)sym; } @@ -170,36 +162,36 @@ HRESULT CCoder::CodeReal(ISequentialInStream *inStream, ISequentialOutStream *ou else { const UInt32 lowDistBits = _inBitStream.ReadBits(numDistDirectBits); - UInt32 dist = _distDecoder.Decode(&_inBitStream); + UInt32 dist = (UInt32)_distDecoder.Decode(&_inBitStream); // if (dist >= kDistTableSize) break; dist = (dist << numDistDirectBits) + lowDistBits; - UInt32 len = _lenDecoder.Decode(&_inBitStream); + unsigned len = _lenDecoder.Decode(&_inBitStream); // if (len >= kLenTableSize) break; if (len == kLenTableSize - 1) len += _inBitStream.ReadBits(kNumLenDirectBits); len += minMatchLen; - { const UInt64 limit = unPackSize - pos; + // limit != 0 if (len > limit) { moreOut = true; len = (UInt32)limit; } } - - while (dist >= pos && len != 0) + do { + // len != 0 + if (dist < pos) + { + _outWindowStream.CopyBlock(dist, len); + pos += len; + break; + } _outWindowStream.PutByte(0); pos++; - len--; - } - - if (len != 0) - { - _outWindowStream.CopyBlock(dist, len); - pos += len; } + while (--len); } } diff --git a/CPP/7zip/Compress/ImplodeDecoder.h b/CPP/7zip/Compress/ImplodeDecoder.h index ec70045..6032515 100644 --- a/CPP/7zip/Compress/ImplodeDecoder.h +++ b/CPP/7zip/Compress/ImplodeDecoder.h @@ -28,7 +28,7 @@ class CHuffmanDecoder Byte _symbols[kMaxHuffTableSize]; public: bool Build(const Byte *lens, unsigned numSymbols) throw(); - UInt32 Decode(CInBit *inStream) const throw(); + unsigned Decode(CInBit *inStream) const throw(); }; @@ -39,6 +39,9 @@ Z7_CLASS_IMP_NOQIB_4( , ICompressSetFinishMode , ICompressGetInStreamProcessedSize ) + Byte _flags; + bool _fullStreamMode; + CLzOutWindow _outWindowStream; CInBit _inBitStream; @@ -46,9 +49,6 @@ Z7_CLASS_IMP_NOQIB_4( CHuffmanDecoder _lenDecoder; CHuffmanDecoder _distDecoder; - Byte _flags; - bool _fullStreamMode; - bool BuildHuff(CHuffmanDecoder &table, unsigned numSymbols); HRESULT CodeReal(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress); diff --git a/CPP/7zip/Compress/LzfseDecoder.cpp b/CPP/7zip/Compress/LzfseDecoder.cpp index 236d5bd..b224a33 100644 --- a/CPP/7zip/Compress/LzfseDecoder.cpp +++ b/CPP/7zip/Compress/LzfseDecoder.cpp @@ -80,7 +80,7 @@ HRESULT CDecoder::DecodeUncompressed(UInt32 unpackSize) UInt32 cur = unpackSize; if (cur > kBufSize) cur = kBufSize; - UInt32 cur2 = (UInt32)m_InStream.ReadBytes(buf, cur); + const UInt32 cur2 = (UInt32)m_InStream.ReadBytes(buf, cur); m_OutWindowStream.PutBytes(buf, cur2); if (cur != cur2) return S_FALSE; @@ -91,7 +91,7 @@ HRESULT CDecoder::DecodeUncompressed(UInt32 unpackSize) HRESULT CDecoder::DecodeLzvn(UInt32 unpackSize, UInt32 packSize) { - PRF(printf("\nLZVN %7u %7u", unpackSize, packSize)); + PRF(printf("\nLZVN 0x%07x 0x%07x\n", unpackSize, packSize)); UInt32 D = 0; @@ -241,19 +241,27 @@ HRESULT CDecoder::DecodeLzvn(UInt32 unpackSize, UInt32 packSize) return S_FALSE; // LZVN encoder writes 7 additional zero bytes - if (packSize != 7) + if (packSize < 7) return S_FALSE; - do + for (unsigned i = 0; i < 7; i++) { Byte b; if (!m_InStream.ReadByte(b)) return S_FALSE; - packSize--; if (b != 0) return S_FALSE; } - while (packSize != 0); - + packSize -= 7; + if (packSize) + { + PRF(printf("packSize after unused = %u\n", packSize)); + // if (packSize <= 0x100) { Byte buf[0x100]; m_InStream.ReadBytes(buf, packSize); } + /* Lzvn block that is used in HFS can contain junk data + (at least 256 bytes) after payload data. Why? + We ignore that junk data, if it's HFS (LzvnMode) mode. */ + if (!LzvnMode) + return S_FALSE; + } return S_OK; } @@ -479,7 +487,7 @@ static Z7_FORCE_INLINE int FseInStream_Init(CBitStream *s, static Z7_FORCE_INLINE UInt32 BitStream_Pull(CBitStream *s, unsigned numBits) { s->numBits -= numBits; - UInt32 v = s->accum >> s->numBits; + const UInt32 v = s->accum >> s->numBits; s->accum = mask31(s->accum, s->numBits); return v; } @@ -922,7 +930,7 @@ HRESULT CDecoder::CodeReal(ISequentialInStream *inStream, ISequentialOutStream * coderReleaser.NeedFlush = false; HRESULT res = m_OutWindowStream.Flush(); if (res == S_OK) - if ((inSize && *inSize != m_InStream.GetProcessedSize()) + if ((!LzvnMode && inSize && *inSize != m_InStream.GetProcessedSize()) || (outSize && *outSize != m_OutWindowStream.GetProcessedSize())) res = S_FALSE; return res; diff --git a/CPP/7zip/Compress/LzhDecoder.cpp b/CPP/7zip/Compress/LzhDecoder.cpp index 064bc55..fb115da 100644 --- a/CPP/7zip/Compress/LzhDecoder.cpp +++ b/CPP/7zip/Compress/LzhDecoder.cpp @@ -10,30 +10,17 @@ namespace NDecoder { static const UInt32 kWindowSizeMin = 1 << 16; -static bool CheckCodeLens(const Byte *lens, unsigned num) -{ - UInt32 sum = 0; - for (unsigned i = 0; i < num; i++) - { - const unsigned len = lens[i]; - if (len != 0) - sum += ((UInt32)1 << (NUM_CODE_BITS - len)); - } - return sum == ((UInt32)1 << NUM_CODE_BITS); -} - bool CCoder::ReadTP(unsigned num, unsigned numBits, int spec) { _symbolT = -1; - const UInt32 n = _inBitStream.ReadBits(numBits); + const unsigned n = (unsigned)_inBitStream.ReadBits(numBits); if (n == 0) { - const unsigned s = _inBitStream.ReadBits(numBits); + const unsigned s = (unsigned)_inBitStream.ReadBits(numBits); _symbolT = (int)s; return (s < num); } - if (n > num) return false; @@ -42,37 +29,31 @@ bool CCoder::ReadTP(unsigned num, unsigned numBits, int spec) unsigned i; for (i = 0; i < NPT; i++) lens[i] = 0; - i = 0; - do { - const UInt32 val = _inBitStream.GetValue(16); + unsigned val = (unsigned)_inBitStream.GetValue(16); unsigned c = val >> 13; - + unsigned mov = 3; if (c == 7) { - UInt32 mask = 1 << 12; - while (mask & val) + while (val & (1 << 12)) { - mask >>= 1; + val += val; c++; } if (c > 16) return false; + mov = c - 3; } - - _inBitStream.MovePos(c < 7 ? 3 : c - 3); lens[i++] = (Byte)c; - - if (i == (unsigned)spec) + _inBitStream.MovePos(mov); + if ((int)i == spec) i += _inBitStream.ReadBits(2); } while (i < n); - if (!CheckCodeLens(lens, NPT)) - return false; - return _decoderT.Build(lens); + return _decoderT.Build(lens, NHuffman::k_BuildMode_Full); } } @@ -82,28 +63,24 @@ bool CCoder::ReadC() { _symbolC = -1; - unsigned n = _inBitStream.ReadBits(NUM_C_BITS); - + const unsigned n = (unsigned)_inBitStream.ReadBits(NUM_C_BITS); if (n == 0) { - const unsigned s = _inBitStream.ReadBits(NUM_C_BITS); + const unsigned s = (unsigned)_inBitStream.ReadBits(NUM_C_BITS); _symbolC = (int)s; return (s < NC); } - if (n > NC) return false; { Byte lens[NC]; - unsigned i = 0; - do { - UInt32 c = (unsigned)_symbolT; + unsigned c = (unsigned)_symbolT; if (_symbolT < 0) - c = _decoderT.Decode(&_inBitStream); + c = _decoderT.DecodeFull(&_inBitStream); if (c <= 2) { @@ -126,19 +103,13 @@ bool CCoder::ReadC() } while (i < n); - while (i < NC) - lens[i++] = 0; - - if (!CheckCodeLens(lens, NC)) - return false; - return _decoderC.Build(lens); + while (i < NC) lens[i++] = 0; + return _decoderC.Build(lens, /* n, */ NHuffman::k_BuildMode_Full); } } -HRESULT CCoder::CodeReal(UInt64 rem, ICompressProgressInfo *progress) +HRESULT CCoder::CodeReal(UInt32 rem, ICompressProgressInfo *progress) { - const unsigned pbit = (DictSize <= (1 << 14) ? 4 : 5); - UInt32 blockSize = 0; while (rem != 0) @@ -147,7 +118,6 @@ HRESULT CCoder::CodeReal(UInt64 rem, ICompressProgressInfo *progress) { if (_inBitStream.ExtraBitsWereRead()) return S_FALSE; - if (progress) { const UInt64 packSize = _inBitStream.GetProcessedSize(); @@ -163,15 +133,16 @@ HRESULT CCoder::CodeReal(UInt64 rem, ICompressProgressInfo *progress) return S_FALSE; if (!ReadC()) return S_FALSE; + const unsigned pbit = (DictSize <= (1 << 14) ? 4 : 5); if (!ReadTP(NP, pbit, -1)) return S_FALSE; } blockSize--; - UInt32 number = (unsigned)_symbolC; + unsigned number = (unsigned)_symbolC; if (_symbolC < 0) - number = _decoderC.Decode(&_inBitStream); + number = _decoderC.DecodeFull(&_inBitStream); if (number < 256) { @@ -180,11 +151,11 @@ HRESULT CCoder::CodeReal(UInt64 rem, ICompressProgressInfo *progress) } else { - UInt32 len = number - 256 + kMatchMinLen; + const unsigned len = number - 256 + kMatchMinLen; - UInt32 dist = (unsigned)_symbolT; + UInt32 dist = (UInt32)(unsigned)_symbolT; if (_symbolT < 0) - dist = _decoderT.Decode(&_inBitStream); + dist = (UInt32)_decoderT.DecodeFull(&_inBitStream); if (dist > 1) { @@ -196,7 +167,11 @@ HRESULT CCoder::CodeReal(UInt64 rem, ICompressProgressInfo *progress) return S_FALSE; if (len > rem) - len = (UInt32)rem; + { + // if (FinishMode) + return S_FALSE; + // len = (unsigned)rem; + } if (!_outWindow.CopyBlock(dist, len)) return S_FALSE; @@ -204,44 +179,37 @@ HRESULT CCoder::CodeReal(UInt64 rem, ICompressProgressInfo *progress) } } - if (FinishMode) + // if (FinishMode) { if (blockSize != 0) return S_FALSE; if (_inBitStream.ReadAlignBits() != 0) return S_FALSE; } - if (_inBitStream.ExtraBitsWereRead()) return S_FALSE; - return S_OK; } -Z7_COM7F_IMF(CCoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, - const UInt64 * /* inSize */, const UInt64 *outSize, ICompressProgressInfo *progress)) +HRESULT CCoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, + const UInt32 outSize, ICompressProgressInfo *progress) { try { - if (!outSize) - return E_INVALIDARG; - if (!_outWindow.Create(DictSize > kWindowSizeMin ? DictSize : kWindowSizeMin)) return E_OUTOFMEMORY; if (!_inBitStream.Create(1 << 17)) return E_OUTOFMEMORY; - _outWindow.SetStream(outStream); _outWindow.Init(false); _inBitStream.SetStream(inStream); _inBitStream.Init(); - - CCoderReleaser coderReleaser(this); - - RINOK(CodeReal(*outSize, progress)) - - coderReleaser.Disable(); + { + CCoderReleaser coderReleaser(this); + RINOK(CodeReal(outSize, progress)) + coderReleaser.Disable(); + } return _outWindow.Flush(); } catch(const CInBufferException &e) { return e.ErrorCode; } diff --git a/CPP/7zip/Compress/LzhDecoder.h b/CPP/7zip/Compress/LzhDecoder.h index 491212e..204c52c 100644 --- a/CPP/7zip/Compress/LzhDecoder.h +++ b/CPP/7zip/Compress/LzhDecoder.h @@ -19,25 +19,25 @@ namespace NDecoder { const unsigned kMatchMinLen = 3; const unsigned kMatchMaxLen = 256; -const unsigned NC = (256 + kMatchMaxLen - kMatchMinLen + 1); +const unsigned NC = 256 + kMatchMaxLen - kMatchMinLen + 1; const unsigned NUM_CODE_BITS = 16; const unsigned NUM_DIC_BITS_MAX = 25; -const unsigned NT = (NUM_CODE_BITS + 3); -const unsigned NP = (NUM_DIC_BITS_MAX + 1); +const unsigned NT = NUM_CODE_BITS + 3; +const unsigned NP = NUM_DIC_BITS_MAX + 1; const unsigned NPT = NP; // Max(NT, NP) -Z7_CLASS_IMP_NOQIB_1( - CCoder - , ICompressCoder -) +class CCoder +{ CLzOutWindow _outWindow; NBitm::CDecoder _inBitStream; int _symbolT; int _symbolC; + UInt32 DictSize; + // bool FinishMode; - NHuffman::CDecoder _decoderT; - NHuffman::CDecoder _decoderC; + NHuffman::CDecoder256 _decoderT; + NHuffman::CDecoder _decoderC; class CCoderReleaser { @@ -52,16 +52,15 @@ Z7_CLASS_IMP_NOQIB_1( bool ReadTP(unsigned num, unsigned numBits, int spec); bool ReadC(); - HRESULT CodeReal(UInt64 outSize, ICompressProgressInfo *progress); + HRESULT CodeReal(UInt32 outSize, ICompressProgressInfo *progress); public: - UInt32 DictSize; - bool FinishMode; - - void SetDictSize(unsigned dictSize) { DictSize = dictSize; } - - CCoder(): DictSize(1 << 16), FinishMode(false) {} - + CCoder(): DictSize(1 << 16) + // , FinishMode(true) + {} + void SetDictSize(UInt32 dictSize) { DictSize = dictSize; } UInt64 GetInputProcessedSize() const { return _inBitStream.GetProcessedSize(); } + HRESULT Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, + UInt32 outSize, ICompressProgressInfo *progress); }; }}} diff --git a/CPP/7zip/Compress/LzmaDecoder.cpp b/CPP/7zip/Compress/LzmaDecoder.cpp index 4f05b48..4b67c8e 100644 --- a/CPP/7zip/Compress/LzmaDecoder.cpp +++ b/CPP/7zip/Compress/LzmaDecoder.cpp @@ -17,6 +17,7 @@ static HRESULT SResToHRESULT(SRes res) case SZ_ERROR_PARAM: return E_INVALIDARG; case SZ_ERROR_UNSUPPORTED: return E_NOTIMPL; case SZ_ERROR_DATA: return S_FALSE; + default: break; } return E_FAIL; } diff --git a/CPP/7zip/Compress/LzmsDecoder.cpp b/CPP/7zip/Compress/LzmsDecoder.cpp index 38c0408..0f6d475 100644 --- a/CPP/7zip/Compress/LzmsDecoder.cpp +++ b/CPP/7zip/Compress/LzmsDecoder.cpp @@ -10,6 +10,61 @@ namespace NCompress { namespace NLzms { +class CBitDecoder +{ +public: + const Byte *_buf; + unsigned _bitPos; + + void Init(const Byte *buf, size_t size) throw() + { + _buf = buf + size; + _bitPos = 0; + } + + Z7_FORCE_INLINE + UInt32 GetValue(unsigned numBits) const + { + UInt32 v = + ((UInt32)_buf[-1] << 16) | + ((UInt32)_buf[-2] << 8) | + (UInt32)_buf[-3]; + v >>= 24 - numBits - _bitPos; + return v & ((1u << numBits) - 1); + } + + Z7_FORCE_INLINE + UInt32 GetValue_InHigh32bits() + { + return GetUi32(_buf - 4) << _bitPos; + } + + void MovePos(unsigned numBits) + { + _bitPos += numBits; + _buf -= (_bitPos >> 3); + _bitPos &= 7; + } + + UInt32 ReadBits32(unsigned numBits) + { + UInt32 mask = (((UInt32)1 << numBits) - 1); + numBits += _bitPos; + const Byte *buf = _buf; + UInt32 v = GetUi32(buf - 4); + if (numBits > 32) + { + v <<= (numBits - 32); + v |= (UInt32)buf[-5] >> (40 - numBits); + } + else + v >>= (32 - numBits); + _buf = buf - (numBits >> 3); + _bitPos = numBits & 7; + return v & mask; + } +}; + static UInt32 g_PosBases[k_NumPosSyms /* + 1 */]; static Byte g_PosDirectBits[k_NumPosSyms]; @@ -91,7 +146,7 @@ static unsigned GetNumPosSlots(size_t size) static const Int32 k_x86_WindowSize = 65535; static const Int32 k_x86_TransOffset = 1023; -static const size_t k_x86_HistorySize = (1 << 16); +static const size_t k_x86_HistorySize = 1 << 16; static void x86_Filter(Byte *data, UInt32 size, Int32 *history) { @@ -318,7 +373,7 @@ HRESULT CDecoder::CodeReal(const Byte *in, size_t inSize, Byte *_win, size_t out { if (_rc.Decode(&mainState, k_NumMainProbs, mainProbs) == 0) { - UInt32 number; + unsigned number; HUFF_DEC(number, m_LitDecoder) LIMIT_CHECK _win[_pos++] = (Byte)number; @@ -330,7 +385,7 @@ HRESULT CDecoder::CodeReal(const Byte *in, size_t inSize, Byte *_win, size_t out if (_rc.Decode(&lzRepStates[0], k_NumRepProbs, lzRepProbs[0]) == 0) { - UInt32 number; + unsigned number; HUFF_DEC(number, m_PosDecoder) LIMIT_CHECK @@ -393,7 +448,7 @@ HRESULT CDecoder::CodeReal(const Byte *in, size_t inSize, Byte *_win, size_t out } } - UInt32 lenSlot; + unsigned lenSlot; HUFF_DEC(lenSlot, m_LenDecoder) LIMIT_CHECK @@ -424,7 +479,7 @@ HRESULT CDecoder::CodeReal(const Byte *in, size_t inSize, Byte *_win, size_t out { UInt64 distance; - UInt32 power; + unsigned power; UInt32 distance32; if (_rc.Decode(&deltaRepStates[0], k_NumRepProbs, deltaRepProbs[0]) == 0) @@ -432,7 +487,7 @@ HRESULT CDecoder::CodeReal(const Byte *in, size_t inSize, Byte *_win, size_t out HUFF_DEC(power, m_PowerDecoder) LIMIT_CHECK - UInt32 number; + unsigned number; HUFF_DEC(number, m_DeltaDecoder) LIMIT_CHECK @@ -502,13 +557,13 @@ HRESULT CDecoder::CodeReal(const Byte *in, size_t inSize, Byte *_win, size_t out const UInt32 dist = (distance32 << power); - UInt32 lenSlot; + unsigned lenSlot; HUFF_DEC(lenSlot, m_LenDecoder) LIMIT_CHECK UInt32 len = g_LenBases[lenSlot]; { - unsigned numDirectBits = k_LenDirectBits[lenSlot]; + const unsigned numDirectBits = k_LenDirectBits[lenSlot]; READ_BITS_CHECK(numDirectBits) len += _bs.ReadBits32(numDirectBits); } diff --git a/CPP/7zip/Compress/LzmsDecoder.h b/CPP/7zip/Compress/LzmsDecoder.h index e173c97..bb21262 100644 --- a/CPP/7zip/Compress/LzmsDecoder.h +++ b/CPP/7zip/Compress/LzmsDecoder.h @@ -4,74 +4,14 @@ #ifndef ZIP7_INC_LZMS_DECODER_H #define ZIP7_INC_LZMS_DECODER_H -// #define SHOW_DEBUG_INFO - -#ifdef SHOW_DEBUG_INFO -#include -#define PRF(x) x -#else -// #define PRF(x) -#endif - #include "../../../C/CpuArch.h" #include "../../../C/HuffEnc.h" -#include "../../Common/MyBuffer.h" -#include "../../Common/MyCom.h" - -#include "../ICoder.h" - #include "HuffmanDecoder.h" namespace NCompress { namespace NLzms { -class CBitDecoder -{ -public: - const Byte *_buf; - unsigned _bitPos; - - void Init(const Byte *buf, size_t size) throw() - { - _buf = buf + size; - _bitPos = 0; - } - - UInt32 GetValue(unsigned numBits) const - { - UInt32 v = ((UInt32)_buf[-1] << 16) | ((UInt32)_buf[-2] << 8) | (UInt32)_buf[-3]; - v >>= (24 - numBits - _bitPos); - return v & ((1 << numBits) - 1); - } - - void MovePos(unsigned numBits) - { - _bitPos += numBits; - _buf -= (_bitPos >> 3); - _bitPos &= 7; - } - - UInt32 ReadBits32(unsigned numBits) - { - UInt32 mask = (((UInt32)1 << numBits) - 1); - numBits += _bitPos; - const Byte *buf = _buf; - UInt32 v = GetUi32(buf - 4); - if (numBits > 32) - { - v <<= (numBits - 32); - v |= (UInt32)buf[-5] >> (40 - numBits); - } - else - v >>= (32 - numBits); - _buf = buf - (numBits >> 3); - _bitPos = numBits & 7; - return v & mask; - } -}; - - const unsigned k_NumLitSyms = 256; const unsigned k_NumLenSyms = 54; const unsigned k_NumPosSyms = 799; @@ -106,18 +46,17 @@ class CHuffDecoder: public NCompress::NHuffman::CDecoderBuildFull(levels, NumSyms); + + this->Build(levels, /* NumSyms, */ NHuffman::k_BuildMode_Full); } void Rebuild() throw() { Generate(); RebuildRem = m_RebuildFreq; - UInt32 num = NumSyms; + const UInt32 num = NumSyms; for (UInt32 i = 0; i < num; i++) Freqs[i] = (Freqs[i] >> 1) + 1; } @@ -197,7 +136,7 @@ struct CRangeDecoder CProbEntry *entry = &probs[st]; st = (st << 1) & (numStates - 1); - UInt32 prob = entry->GetProb(); + const UInt32 prob = entry->GetProb(); if (range <= 0xFFFF) { @@ -208,7 +147,7 @@ struct CRangeDecoder cur += 2; } - UInt32 bound = (range >> k_NumProbBits) * prob; + const UInt32 bound = (range >> k_NumProbBits) * prob; if (code < bound) { @@ -232,7 +171,6 @@ struct CRangeDecoder class CDecoder { // CRangeDecoder _rc; - // CBitDecoder _bs; size_t _pos; UInt32 _reps[k_NumReps + 1]; diff --git a/CPP/7zip/Compress/Lzx.h b/CPP/7zip/Compress/Lzx.h index 2532088..39fca0c 100644 --- a/CPP/7zip/Compress/Lzx.h +++ b/CPP/7zip/Compress/Lzx.h @@ -52,7 +52,9 @@ const unsigned kNumDictBits_Max = 21; const UInt32 kDictSize_Max = (UInt32)1 << kNumDictBits_Max; const unsigned kNumLinearPosSlotBits = 17; -const unsigned kNumPowerPosSlots = 38; +// const unsigned kNumPowerPosSlots = 38; +// const unsigned kNumPowerPosSlots = (kNumLinearPosSlotBits + 1) * 2; // non-including two first linear slots. +const unsigned kNumPowerPosSlots = (kNumLinearPosSlotBits + 2) * 2; // including two first linear slots. }} diff --git a/CPP/7zip/Compress/LzxDecoder.cpp b/CPP/7zip/Compress/LzxDecoder.cpp index b50d863..a7b9986 100644 --- a/CPP/7zip/Compress/LzxDecoder.cpp +++ b/CPP/7zip/Compress/LzxDecoder.cpp @@ -3,6 +3,7 @@ #include "StdAfx.h" #include +// #include // #define SHOW_DEBUG_INFO @@ -14,448 +15,1433 @@ #endif #include "../../../C/Alloc.h" +#include "../../../C/RotateDefs.h" +#include "../../../C/CpuArch.h" #include "LzxDecoder.h" + +#ifdef MY_CPU_X86_OR_AMD64 +#if defined(MY_CPU_AMD64) \ + || defined(__SSE2__) \ + || defined(_M_IX86_FP) && (_M_IX86_FP >= 2) \ + || 0 && defined(_MSC_VER) && (_MSC_VER >= 1400) // set (1 &&) for debug + +#if defined(__clang__) && (__clang_major__ >= 2) \ + || defined(__GNUC__) && (__GNUC__ >= 4) \ + || defined(_MSC_VER) && (_MSC_VER >= 1400) +#define Z7_LZX_X86_FILTER_USE_SSE2 +#endif +#endif +#endif + + +#ifdef Z7_LZX_X86_FILTER_USE_SSE2 +// #ifdef MY_CPU_X86_OR_AMD64 +#include // SSE2 +// #endif + #if defined(__clang__) || defined(__GNUC__) + typedef int ctz_type; + #define MY_CTZ(dest, mask) dest = __builtin_ctz((UInt32)(mask)) + #else // #if defined(_MSC_VER) + #if (_MSC_VER >= 1600) + // #include + #endif + typedef unsigned long ctz_type; + #define MY_CTZ(dest, mask) _BitScanForward(&dest, (mask)); + #endif // _MSC_VER +#endif + +// when window buffer is filled, we must wrap position to zero, +// and we want to wrap at same points where original-lzx must wrap. +// But the wrapping is possible in point where chunk is finished. +// Usually (chunk_size == 32KB), but (chunk_size != 32KB) also is allowed. +// So we don't use additional buffer space over required (winSize). +// And we can't use large overwrite after (len) in CopyLzMatch(). +// But we are allowed to write 3 bytes after (len), because +// (delta <= _winSize - 3). + +// #define k_Lz_OverwriteSize 0 // for debug : to disable overwrite +#define k_Lz_OverwriteSize 3 // = kNumReps +#if k_Lz_OverwriteSize > 0 +// (k_Lz_OutBufSize_Add >= k_Lz_OverwriteSize) is required +// we use value 4 to simplify memset() code. +#define k_Lz_OutBufSize_Add (k_Lz_OverwriteSize + 1) // == 4 +#else +#define k_Lz_OutBufSize_Add 0 +#endif + +// (len != 0) +// (0 < delta <= _winSize - 3) +Z7_FORCE_INLINE +void CopyLzMatch(Byte *dest, const Byte *src, UInt32 len, UInt32 delta) +{ + if (delta >= 4) + { +#if k_Lz_OverwriteSize >= 3 + // optimized code with overwrite to reduce the number of branches + #ifdef MY_CPU_LE_UNALIGN + *(UInt32 *)(void *)(dest) = *(const UInt32 *)(const void *)(src); + #else + dest[0] = src[0]; + dest[1] = src[1]; + dest[2] = src[2]; + dest[3] = src[3]; + #endif + len--; + src++; + dest++; + { +#else + // no overwrite in out buffer + dest[0] = src[0]; + { + const unsigned m = (unsigned)len & 1; + src += m; + dest += m; + } + if (len &= ~(unsigned)1) + { + dest[0] = src[0]; + dest[1] = src[1]; +#endif + // len == 0 is allowed here + { + const unsigned m = (unsigned)len & 3; + src += m; + dest += m; + } + if (len &= ~(unsigned)3) + { +#ifdef MY_CPU_LE_UNALIGN + #if 1 + *(UInt32 *)(void *)(dest) = *(const UInt32 *)(const void *)(src); + { + const unsigned m = (unsigned)len & 7; + dest += m; + src += m; + } + if (len &= ~(unsigned)7) + do + { + *(UInt32 *)(void *)(dest ) = *(const UInt32 *)(const void *)(src); + *(UInt32 *)(void *)(dest + 4) = *(const UInt32 *)(const void *)(src + 4); + src += 8; + dest += 8; + } + while (len -= 8); + #else + // gcc-11 -O3 for x64 generates incorrect code here + do + { + *(UInt32 *)(void *)(dest) = *(const UInt32 *)(const void *)(src); + src += 4; + dest += 4; + } + while (len -= 4); + #endif +#else + do + { + const Byte b0 = src[0]; + const Byte b1 = src[1]; + dest[0] = b0; + dest[1] = b1; + const Byte b2 = src[2]; + const Byte b3 = src[3]; + dest[2] = b2; + dest[3] = b3; + src += 4; + dest += 4; + } + while (len -= 4); +#endif + } + } + } + else // (delta < 4) + { + const unsigned b0 = *src; + *dest = (Byte)b0; + if (len >= 2) + { + if (delta < 2) + { + dest += (unsigned)len & 1; + dest[0] = (Byte)b0; + dest[1] = (Byte)b0; + dest += (unsigned)len & 2; + if (len &= ~(unsigned)3) + { +#ifdef MY_CPU_LE_UNALIGN + #ifdef MY_CPU_64BIT + const UInt64 a = (UInt64)b0 * 0x101010101010101; + *(UInt32 *)(void *)dest = (UInt32)a; + dest += (unsigned)len & 7; + if (len &= ~(unsigned)7) + { + // *(UInt64 *)(void *)dest = a; + // dest += 8; + // len -= 8; + // if (len) + { + // const ptrdiff_t delta = (ptrdiff_t)dest & 7; + // dest -= delta; + do + { + *(UInt64 *)(void *)dest = a; + dest += 8; + } + while (len -= 8); + // dest += delta - 8; + // *(UInt64 *)(void *)dest = a; + } + } + #else + const UInt32 a = (UInt32)b0 * 0x1010101; + do + { + *(UInt32 *)(void *)dest = a; + dest += 4; + } + while (len -= 4); + #endif +#else + do + { + dest[0] = (Byte)b0; + dest[1] = (Byte)b0; + dest[2] = (Byte)b0; + dest[3] = (Byte)b0; + dest += 4; + } + while (len -= 4); +#endif + } + } + else if (delta == 2) + { + const unsigned m = (unsigned)len & 1; + len &= ~(unsigned)1; + src += m; + dest += m; + { + const Byte a0 = src[0]; + const Byte a1 = src[1]; + do + { + dest[0] = a0; + dest[1] = a1; + dest += 2; + } + while (len -= 2); + } + } + else /* if (delta == 3) */ + { + const unsigned b1 = src[1]; + dest[1] = (Byte)b1; + if (len -= 2) + { + const unsigned b2 = src[2]; + dest += 2; + do + { + dest[0] = (Byte)b2; if (--len == 0) break; + dest[1] = (Byte)b0; if (--len == 0) break; + dest[2] = (Byte)b1; + dest += 3; + } + while (--len); + } + } + } + } +} + +// #define Z7_LZX_SHOW_STAT +#ifdef Z7_LZX_SHOW_STAT +#include +#endif + namespace NCompress { namespace NLzx { -static void x86_Filter(Byte *data, UInt32 size, UInt32 processedSize, UInt32 translationSize) +// #define Z7_LZX_SHOW_STAT +#ifdef Z7_LZX_SHOW_STAT +static UInt32 g_stats_Num_x86[3]; +static UInt32 g_stats_NumTables; +static UInt32 g_stats_NumLits; +static UInt32 g_stats_NumAlign; +static UInt32 g_stats_main[kMainTableSize]; +static UInt32 g_stats_len[kNumLenSymbols]; +static UInt32 g_stats_main_levels[kNumHuffmanBits + 1]; +static UInt32 g_stats_len_levels[kNumHuffmanBits + 1]; +#define UPDATE_STAT(a) a +static void PrintVal(UInt32 v) +{ + printf("\n : %9u", v); +} +static void PrintStat(const char *name, const UInt32 *a, size_t num) +{ + printf("\n\n==== %s:", name); + UInt32 sum = 0; + size_t i; + for (i = 0; i < num; i++) + sum += a[i]; + PrintVal(sum); + if (sum != 0) + { + for (i = 0; i < num; i++) + { + if (i % 8 == 0) + printf("\n"); + printf("\n%3x : %9u : %5.2f", (unsigned)i, (unsigned)a[i], (double)a[i] * 100 / sum); + } + } + printf("\n"); +} + +static struct CStat { - const UInt32 kResidue = 10; + ~CStat() + { + PrintStat("x86_filter", g_stats_Num_x86, Z7_ARRAY_SIZE(g_stats_Num_x86)); + printf("\nTables:"); PrintVal(g_stats_NumTables); + printf("\nLits:"); PrintVal(g_stats_NumLits); + printf("\nAlign:"); PrintVal(g_stats_NumAlign); + PrintStat("Main", g_stats_main, Z7_ARRAY_SIZE(g_stats_main)); + PrintStat("Len", g_stats_len, Z7_ARRAY_SIZE(g_stats_len)); + PrintStat("Main Levels", g_stats_main_levels, Z7_ARRAY_SIZE(g_stats_main_levels)); + PrintStat("Len Levels", g_stats_len_levels, Z7_ARRAY_SIZE(g_stats_len_levels)); + } +} g_stat; +#else +#define UPDATE_STAT(a) +#endif + + + +/* +3 p015 : ivb- : or r32,r32 / add r32,r32 +4 p0156 : hsw+ +5 p0156b: adl+ +2 p0_5 : ivb- : shl r32,i8 +2 p0__6 : hsw+ +1 p5 : ivb- : jb +2 p0__6 : hsw+ +2 p0_5 : wsm- : SSE2 : pcmpeqb : _mm_cmpeq_epi8 +2 p_15 : snb-bdw +2 p01 : skl+ +1 p0 : SSE2 : pmovmskb : _mm_movemask_epi8 +*/ +/* + v24.00: the code was fixed for more compatibility with original-ms-cab-decoder. + for ((Int32)translationSize >= 0) : LZX specification shows the code with signed Int32. + for ((Int32)translationSize < 0) : no specification for that case, but we support that case. + We suppose our code now is compatible with original-ms-cab-decoder. + + Starting byte of data stream (real_pos == 0) is special corner case, + where we don't need any conversion (as in original-ms-cab-decoder). + Our optimization: we use unsigned (UInt32 pos) (pos = -1 - real_pos). + So (pos) is always negative: ((Int32)pos < 0). + It allows us to use simple comparison (v > pos) instead of more complex comparisons. +*/ +// (p) will point 5 bytes after 0xe8 byte: +// pos == -1 - (p - 5 - data_start) == 4 + data_start - p +// (FILTER_PROCESSED_SIZE_DELTA == 4) is optimized value for better speed in some compilers: +#define FILTER_PROCESSED_SIZE_DELTA 4 + +#if defined(MY_CPU_X86_OR_AMD64) || defined(MY_CPU_ARM_OR_ARM64) + // optimized branch: + // size_t must be at least 32-bit for this branch. + #if 1 // use 1 for simpler code + // use integer (low 32 bits of pointer) instead of pointer + #define X86_FILTER_PREPARE processedSize4 = (UInt32)(size_t)(ptrdiff_t)data + \ + (UInt32)(4 - FILTER_PROCESSED_SIZE_DELTA) - processedSize4; + #define X86_FILTER_CALC_pos(p) const UInt32 pos = processedSize4 - (UInt32)(size_t)(ptrdiff_t)p; + #else + // note: (dataStart) pointer can point out of array ranges: + #define X86_FILTER_PREPARE const Byte *dataStart = data + \ + (4 - FILTER_PROCESSED_SIZE_DELTA) - processedSize4; + #define X86_FILTER_CALC_pos(p) const UInt32 pos = (UInt32)(size_t)(dataStart - p); + #endif +#else + // non-optimized branch for unusual platforms (16-bit size_t or unusual size_t): + #define X86_FILTER_PREPARE processedSize4 = \ + (UInt32)(4 - FILTER_PROCESSED_SIZE_DELTA) - processedSize4; + #define X86_FILTER_CALC_pos(p) const UInt32 pos = processedSize4 - (UInt32)(size_t)(p - data); +#endif + +#define X86_TRANSLATE_PRE(p) \ + UInt32 v = GetUi32((p) - 4); + +#define X86_TRANSLATE_POST(p) \ + { \ + X86_FILTER_CALC_pos(p) \ + if (v < translationSize) { \ + UPDATE_STAT(g_stats_Num_x86[0]++;) \ + v += pos + 1; \ + SetUi32((p) - 4, v) \ + } \ + else if (v > pos) { \ + UPDATE_STAT(g_stats_Num_x86[1]++;) \ + v += translationSize; \ + SetUi32((p) - 4, v) \ + } else { UPDATE_STAT(g_stats_Num_x86[2]++;) } \ + } + + +/* + if ( defined(Z7_LZX_X86_FILTER_USE_SSE2) + && defined(Z7_LZX_X86_FILTER_USE_SSE2_ALIGNED)) + the function can read up to aligned_for_32_up_from(size) bytes in (data). +*/ +// processedSize < (1 << 30) +Z7_NO_INLINE +static void x86_Filter4(Byte *data, size_t size, UInt32 processedSize4, UInt32 translationSize) +{ + const size_t kResidue = 10; if (size <= kResidue) return; - size -= kResidue; - - const Byte save = data[(size_t)size + 4]; - data[(size_t)size + 4] = 0xE8; - - for (UInt32 i = 0;;) + Byte * const lim = data + size - kResidue + 4; + const Byte save = lim[0]; + lim[0] = 0xe8; + X86_FILTER_PREPARE + Byte *p = data; + +#define FILTER_RETURN_IF_LIM(_p_) if (_p_ > lim) { lim[0] = save; return; } + +#ifdef Z7_LZX_X86_FILTER_USE_SSE2 + +// sse2-aligned/sse2-unaligned provide same speed on real data. +// but the code is smaller for sse2-unaligned version. +// for debug : define it to get alternative version with aligned 128-bit reads: +// #define Z7_LZX_X86_FILTER_USE_SSE2_ALIGNED + +#define FILTER_MASK_INT UInt32 +#define FILTER_NUM_VECTORS_IN_CHUNK 2 +#define FILTER_CHUNK_BYTES_OFFSET (16 * FILTER_NUM_VECTORS_IN_CHUNK - 5) + +#ifdef Z7_LZX_X86_FILTER_USE_SSE2_ALIGNED + // aligned version doesn't uses additional space if buf size is aligned for 32 + #define k_Filter_OutBufSize_Add 0 + #define k_Filter_OutBufSize_AlignMask (16 * FILTER_NUM_VECTORS_IN_CHUNK - 1) + #define FILTER_LOAD_128(p) _mm_load_si128 ((const __m128i *)(const void *)(p)) +#else + #define k_Filter_OutBufSize_Add (16 * FILTER_NUM_VECTORS_IN_CHUNK) + #define k_Filter_OutBufSize_AlignMask 0 + #define FILTER_LOAD_128(p) _mm_loadu_si128((const __m128i *)(const void *)(p)) +#endif + +#define GET_E8_MASK(dest, dest1, p) \ +{ \ + __m128i v0 = FILTER_LOAD_128(p); \ + __m128i v1 = FILTER_LOAD_128(p + 16); \ + p += 16 * FILTER_NUM_VECTORS_IN_CHUNK; \ + v0 = _mm_cmpeq_epi8(v0, k_e8_Vector); \ + v1 = _mm_cmpeq_epi8(v1, k_e8_Vector); \ + dest = (unsigned)_mm_movemask_epi8(v0); \ + dest1 = (unsigned)_mm_movemask_epi8(v1); \ +} + + const __m128i k_e8_Vector = _mm_set1_epi32((Int32)(UInt32)0xe8e8e8e8); + for (;;) { - Byte *p = data + i; + // for debug: define it for smaller code: + // #define Z7_LZX_X86_FILTER_CALC_IN_LOOP + // without Z7_LZX_X86_FILTER_CALC_IN_LOOP, we can get faster and simpler loop + FILTER_MASK_INT mask; + { + FILTER_MASK_INT mask1; + do + { + GET_E8_MASK(mask, mask1, p) + #ifndef Z7_LZX_X86_FILTER_CALC_IN_LOOP + mask += mask1; + #else + mask |= mask1 << 16; + #endif + } + while (!mask); + + #ifndef Z7_LZX_X86_FILTER_CALC_IN_LOOP + mask -= mask1; + mask |= mask1 << 16; + #endif + } + +#ifdef Z7_LZX_X86_FILTER_USE_SSE2_ALIGNED for (;;) { - if (*p++ == 0xE8) break; - if (*p++ == 0xE8) break; - if (*p++ == 0xE8) break; - if (*p++ == 0xE8) break; + ctz_type index; + typedef + #ifdef MY_CPU_64BIT + UInt64 + #else + UInt32 + #endif + SUPER_MASK_INT; + SUPER_MASK_INT superMask; + { + MY_CTZ(index, mask); + Byte *p2 = p - FILTER_CHUNK_BYTES_OFFSET + (unsigned)index; + X86_TRANSLATE_PRE(p2) + superMask = ~(SUPER_MASK_INT)0x1f << index; + FILTER_RETURN_IF_LIM(p2) + X86_TRANSLATE_POST(p2) + mask &= (UInt32)superMask; + } + if (mask) + continue; + if (index <= FILTER_CHUNK_BYTES_OFFSET) + break; + { + FILTER_MASK_INT mask1; + GET_E8_MASK(mask, mask1, p) + mask &= + #ifdef MY_CPU_64BIT + (UInt32)(superMask >> 32); + #else + ((FILTER_MASK_INT)0 - 1) << ((int)index - FILTER_CHUNK_BYTES_OFFSET); + #endif + mask |= mask1 << 16; + } + if (!mask) + break; } - - i = (UInt32)(p - data); - - if (i > size) - break; +#else // ! Z7_LZX_X86_FILTER_USE_SSE2_ALIGNED { - Int32 v = (Int32)GetUi32(p); - Int32 pos = (Int32)((Int32)1 - (Int32)(processedSize + i)); - i += 4; - if (v >= pos && v < (Int32)translationSize) + // we use simplest version without loop: + // for (;;) { - v += (v >= 0 ? pos : (Int32)translationSize); - SetUi32(p, (UInt32)v) + ctz_type index; + MY_CTZ(index, mask); + /* + printf("\np=%p, mask=%8x, index = %2d, p + index = %x\n", + (p - 16 * FILTER_NUM_VECTORS_IN_CHUNK), (unsigned)mask, + (unsigned)index, (unsigned)((unsigned)(ptrdiff_t)(p - 16 * FILTER_NUM_VECTORS_IN_CHUNK) + index)); + */ + p += (size_t)(unsigned)index - FILTER_CHUNK_BYTES_OFFSET; + FILTER_RETURN_IF_LIM(p) + // mask &= ~(FILTER_MASK_INT)0x1f << index; mask >>= index; + X86_TRANSLATE_PRE(p) + X86_TRANSLATE_POST(p) + // if (!mask) break; // p += 16 * FILTER_NUM_VECTORS_IN_CHUNK; } } +#endif // ! Z7_LZX_X86_FILTER_USE_SSE2_ALIGNED } - data[(size_t)size + 4] = save; +#else // ! Z7_LZX_X86_FILTER_USE_SSE2 + +#define k_Filter_OutBufSize_Add 0 +#define k_Filter_OutBufSize_AlignMask 0 + + + for (;;) + { + for (;;) + { + if (p[0] == 0xe8) { p += 5; break; } + if (p[1] == 0xe8) { p += 6; break; } + if (p[2] == 0xe8) { p += 7; break; } + p += 4; + if (p[-1] == 0xe8) { p += 4; break; } + } + FILTER_RETURN_IF_LIM(p) + X86_TRANSLATE_PRE(p) + X86_TRANSLATE_POST(p) + } + +#endif // ! Z7_LZX_X86_FILTER_USE_SSE2 } -CDecoder::CDecoder(bool wimMode): +CDecoder::CDecoder() throw(): _win(NULL), + _isUncompressedBlock(false), _skipByte(false), - _unpackBlockSize(0), - KeepHistoryForNext(true), - NeedAlloc(true), _keepHistory(false), - _wimMode(wimMode), + _keepHistoryForNext(true), + _needAlloc(true), + _wimMode(false), _numDictBits(15), - _x86_buf(NULL), + _unpackBlockSize(0), _x86_translationSize(0), + _x86_buf(NULL), _unpackedData(NULL) { + { + // it's better to get empty virtual entries, if mispredicted value can be used: + memset(_reps, 0, kPosSlotOffset * sizeof(_reps[0])); + memset(_extra, 0, kPosSlotOffset); +#define SET_NUM_BITS(i) i // #define NUM_BITS_DELTA 31 + _extra[kPosSlotOffset + 0] = SET_NUM_BITS(0); + _extra[kPosSlotOffset + 1] = SET_NUM_BITS(0); + // reps[0] = 0 - (kNumReps - 1); + // reps[1] = 1 - (kNumReps - 1); + UInt32 a = 2 - (kNumReps - 1); + UInt32 delta = 1; + unsigned i; + for (i = 0; i < kNumLinearPosSlotBits; i++) + { + _extra[(size_t)i * 2 + 2 + kPosSlotOffset] = (Byte)(SET_NUM_BITS(i)); + _extra[(size_t)i * 2 + 3 + kPosSlotOffset] = (Byte)(SET_NUM_BITS(i)); + _reps [(size_t)i * 2 + 2 + kPosSlotOffset] = a; a += delta; + _reps [(size_t)i * 2 + 3 + kPosSlotOffset] = a; a += delta; + delta += delta; + } + for (i = kNumLinearPosSlotBits * 2 + 2; i < kNumPosSlots; i++) + { + _extra[(size_t)i + kPosSlotOffset] = SET_NUM_BITS(kNumLinearPosSlotBits); + _reps [(size_t)i + kPosSlotOffset] = a; + a += (UInt32)1 << kNumLinearPosSlotBits; + } + } } -CDecoder::~CDecoder() +CDecoder::~CDecoder() throw() { - if (NeedAlloc) - ::MidFree(_win); - ::MidFree(_x86_buf); + if (_needAlloc) + // BigFree + z7_AlignedFree + (_win); + z7_AlignedFree(_x86_buf); } -HRESULT CDecoder::Flush() +HRESULT CDecoder::Flush() throw() { + // UInt32 t = _x86_processedSize; for (int y = 0; y < 50; y++) { _x86_processedSize = t; // benchmark: (branch predicted) if (_x86_translationSize != 0) { Byte *destData = _win + _writePos; const UInt32 curSize = _pos - _writePos; - if (KeepHistoryForNext) + if (_keepHistoryForNext) { + const size_t kChunkSize = (size_t)1 << 15; + if (curSize > kChunkSize) + return E_NOTIMPL; if (!_x86_buf) { - // we must change it to support another chunk sizes - const size_t kChunkSize = (size_t)1 << 15; - if (curSize > kChunkSize) - return E_NOTIMPL; - _x86_buf = (Byte *)::MidAlloc(kChunkSize); + // (kChunkSize % 32 == 0) is required in some cases, because + // the filter can read data by 32-bytes chunks in some cases. + // if (chunk_size > (1 << 15)) is possible, then we must the code: + const size_t kAllocSize = kChunkSize + k_Filter_OutBufSize_Add; + _x86_buf = (Byte *)z7_AlignedAlloc(kAllocSize); if (!_x86_buf) return E_OUTOFMEMORY; + #if 0 != k_Filter_OutBufSize_Add || \ + 0 != k_Filter_OutBufSize_AlignMask + // x86_Filter4() can read after curSize. + // So we set all data to zero to prevent reading of uninitialized data: + memset(_x86_buf, 0, kAllocSize); // optional + #endif } + // for (int yy = 0; yy < 1; yy++) // for debug memcpy(_x86_buf, destData, curSize); _unpackedData = _x86_buf; destData = _x86_buf; } - x86_Filter(destData, (UInt32)curSize, _x86_processedSize, _x86_translationSize); + else + { + // x86_Filter4() can overread after (curSize), + // so we can do memset() after (curSize): + // k_Filter_OutBufSize_AlignMask also can be used + // if (!_overDict) memset(destData + curSize, 0, k_Filter_OutBufSize_Add); + } + x86_Filter4(destData, curSize, _x86_processedSize - FILTER_PROCESSED_SIZE_DELTA, _x86_translationSize); _x86_processedSize += (UInt32)curSize; if (_x86_processedSize >= ((UInt32)1 << 30)) _x86_translationSize = 0; } - + // } return S_OK; } -UInt32 CDecoder::ReadBits(unsigned numBits) { return _bitStream.ReadBitsSmall(numBits); } + +// (NUM_DELTA_BYTES == 2) reduces the code in main loop. +#if 1 + #define NUM_DELTA_BYTES 2 +#else + #define NUM_DELTA_BYTES 0 +#endif + +#define NUM_DELTA_BIT_OFFSET_BITS (NUM_DELTA_BYTES * 8) + +#if NUM_DELTA_BIT_OFFSET_BITS > 0 + #define DECODE_ERROR_CODE 0 + #define IS_OVERFLOW_bitOffset(bo) ((bo) >= 0) + // ( >= 0) comparison after bitOffset change gives simpler commands than ( > 0) comparison +#else + #define DECODE_ERROR_CODE 1 + #define IS_OVERFLOW_bitOffset(bo) ((bo) > 0) +#endif + +// (numBits != 0) +#define GET_VAL_BASE(numBits) (_value >> (32 - (numBits))) + +#define Z7_LZX_HUFF_DECODE( sym, huff, kNumTableBits, move_pos_op, check_op, error_op) \ + Z7_HUFF_DECODE_VAL_IN_HIGH32(sym, huff, kNumHuffmanBits, kNumTableBits, \ + _value, check_op, error_op, move_pos_op, NORMALIZE, bs) + +#define Z7_LZX_HUFF_DECODE_CHECK_YES(sym, huff, kNumTableBits, move_pos_op) \ + Z7_LZX_HUFF_DECODE( sym, huff, kNumTableBits, move_pos_op, \ + Z7_HUFF_DECODE_ERROR_SYM_CHECK_YES, { return DECODE_ERROR_CODE; }) + +#define Z7_LZX_HUFF_DECODE_CHECK_NO( sym, huff, kNumTableBits, move_pos_op) \ + Z7_LZX_HUFF_DECODE( sym, huff, kNumTableBits, move_pos_op, \ + Z7_HUFF_DECODE_ERROR_SYM_CHECK_NO, {}) + +#define NORMALIZE \ +{ \ + const Byte *ptr = _buf + (_bitOffset >> 4) * 2; \ + /* _value = (((UInt32)GetUi16(ptr) << 16) | GetUi16(ptr + 2)) << (_bitOffset & 15); */ \ + const UInt32 v = GetUi32(ptr); \ + _value = rotlFixed (v, ((int)_bitOffset & 15) + 16); \ +} + +#define MOVE_POS(bs, numBits) \ +{ \ + _bitOffset += numBits; \ +} + +#define MOVE_POS_STAT(bs, numBits) \ +{ \ + UPDATE_STAT(g_stats_len_levels[numBits]++;) \ + MOVE_POS(bs, numBits); \ +} + +#define MOVE_POS_CHECK(bs, numBits) \ +{ \ + if (IS_OVERFLOW_bitOffset(_bitOffset += numBits)) return DECODE_ERROR_CODE; \ +} + +#define MOVE_POS_CHECK_STAT(bs, numBits) \ +{ \ + UPDATE_STAT(g_stats_main_levels[numBits]++;) \ + MOVE_POS_CHECK(bs, numBits) \ +} + + +// (numBits == 0) is supported + +#ifdef Z7_HUFF_USE_64BIT_LIMIT + +#define MACRO_ReadBitsBig_pre(numBits) \ +{ \ + _bitOffset += (numBits); \ + _value >>= 32 - (numBits); \ +} + +#else + +#define MACRO_ReadBitsBig_pre(numBits) \ +{ \ + _bitOffset += (numBits); \ + _value = (UInt32)((UInt32)_value >> 1 >> (31 ^ (numBits))); \ +} + +#endif + + +#define MACRO_ReadBitsBig_add(dest) \ + { dest += (UInt32)_value; } + +#define MACRO_ReadBitsBig_add3(dest) \ + { dest += (UInt32)(_value) << 3; } + + +// (numBits != 0) +#define MACRO_ReadBits_NonZero(val, numBits) \ +{ \ + val = (UInt32)(_value >> (32 - (numBits))); \ + MOVE_POS(bs, numBits); \ + NORMALIZE \ +} + + +struct CBitDecoder +{ + ptrdiff_t _bitOffset; + const Byte *_buf; + + Z7_FORCE_INLINE + UInt32 GetVal() const + { + const Byte *ptr = _buf + (_bitOffset >> 4) * 2; + const UInt32 v = GetUi32(ptr); + return rotlFixed (v, ((int)_bitOffset & 15) + 16); + } + + Z7_FORCE_INLINE + bool IsOverRead() const + { + return _bitOffset > (int)(0 - NUM_DELTA_BIT_OFFSET_BITS); + } + + + Z7_FORCE_INLINE + bool WasBitStreamFinishedOK() const + { + // we check that all 0-15 unused bits are zeros: + if (_bitOffset == 0 - NUM_DELTA_BIT_OFFSET_BITS) + return true; + if ((_bitOffset + NUM_DELTA_BIT_OFFSET_BITS + 15) & ~(ptrdiff_t)15) + return false; + const Byte *ptr = _buf - NUM_DELTA_BYTES - 2; + if ((UInt16)(GetUi16(ptr) << (_bitOffset & 15))) + return false; + return true; + } + + // (numBits != 0) + Z7_FORCE_INLINE + UInt32 ReadBits_NonZero(unsigned numBits) throw() + { + const UInt32 val = GetVal() >> (32 - numBits); + _bitOffset += numBits; + return val; + } +}; + + +class CBitByteDecoder: public CBitDecoder +{ + size_t _size; +public: + + Z7_FORCE_INLINE + void Init_ByteMode(const Byte *data, size_t size) + { + _buf = data; + _size = size; + } + + Z7_FORCE_INLINE + void Init_BitMode(const Byte *data, size_t size) + { + _size = size & 1; + size &= ~(size_t)1; + _buf = data + size + NUM_DELTA_BYTES; + _bitOffset = 0 - (ptrdiff_t)(size * 8) - NUM_DELTA_BIT_OFFSET_BITS; + } + + Z7_FORCE_INLINE + void Switch_To_BitMode() + { + Init_BitMode(_buf, _size); + } + + Z7_FORCE_INLINE + bool Switch_To_ByteMode() + { + /* here we check that unused bits in high 16-bits word are zeros. + If high word is full (all 16-bits are unused), + we check that all 16-bits are zeros. + So we check and skip (1-16 bits) unused bits */ + if ((GetVal() >> (16 + (_bitOffset & 15))) != 0) + return false; + _bitOffset += 16; + _bitOffset &= ~(ptrdiff_t)15; + if (_bitOffset > 0 - NUM_DELTA_BIT_OFFSET_BITS) + return false; + const ptrdiff_t delta = _bitOffset >> 3; + _size = (size_t)((ptrdiff_t)(_size) - delta - NUM_DELTA_BYTES); + _buf += delta; + // _bitOffset = 0; // optional + return true; + } + + Z7_FORCE_INLINE + size_t GetRem() const { return _size; } + + Z7_FORCE_INLINE + UInt32 ReadUInt32() + { + const Byte *ptr = _buf; + const UInt32 v = GetUi32(ptr); + _buf += 4; + _size -= 4; + return v; + } + + Z7_FORCE_INLINE + void CopyTo(Byte *dest, size_t size) + { + memcpy(dest, _buf, size); + _buf += size; + _size -= size; + } + + Z7_FORCE_INLINE + bool IsOneDirectByteLeft() const + { + return GetRem() == 1; + } + + Z7_FORCE_INLINE + Byte DirectReadByte() + { + _size--; + return *_buf++; + } +}; + + +// numBits != 0 +// Z7_FORCE_INLINE +Z7_NO_INLINE +static +UInt32 ReadBits(CBitDecoder &_bitStream, unsigned numBits) +{ + return _bitStream.ReadBits_NonZero(numBits); +} #define RIF(x) { if (!(x)) return false; } -bool CDecoder::ReadTable(Byte *levels, unsigned numSymbols) + +/* +MSVC compiler adds extra move operation, + if we access array with 32-bit index + array[calc_index_32_bit(32-bit_var)] + where calc_index_32_bit operations are: ((unsigned)a>>cnt), &, ^, | + clang is also affected for ((unsigned)a>>cnt) in byte array. +*/ + +// it can overread input buffer for 7-17 bytes. +// (levels != levelsEnd) +Z7_NO_INLINE +static ptrdiff_t ReadTable(ptrdiff_t _bitOffset, const Byte *_buf, Byte *levels, const Byte *levelsEnd) { + const unsigned kNumTableBits_Level = 7; + NHuffman::CDecoder256 _levelDecoder; + NHuffman::CValueInt _value; + // optional check to reduce size of overread zone: + if (_bitOffset > (int)0 - (int)NUM_DELTA_BIT_OFFSET_BITS - (int)(kLevelTableSize * kNumLevelBits)) + return DECODE_ERROR_CODE; + NORMALIZE { - Byte levels2[kLevelTableSize]; - for (unsigned i = 0; i < kLevelTableSize; i++) - levels2[i] = (Byte)ReadBits(kNumLevelBits); - RIF(_levelDecoder.Build(levels2)) + Byte levels2[kLevelTableSize / 4 * 4]; + for (size_t i = 0; i < kLevelTableSize / 4 * 4; i += 4) + { + UInt32 val; + MACRO_ReadBits_NonZero(val, kNumLevelBits * 4) + levels2[i + 0] = (Byte)((val >> (3 * kNumLevelBits))); + levels2[i + 1] = (Byte)((val >> (2 * kNumLevelBits)) & ((1u << kNumLevelBits) - 1)); + levels2[i + 2] = (Byte)((Byte)val >> (1 * kNumLevelBits)); + levels2[i + 3] = (Byte)((val) & ((1u << kNumLevelBits) - 1)); + } + RIF(_levelDecoder.Build(levels2, NHuffman::k_BuildMode_Full)) } - unsigned i = 0; do { - UInt32 sym = _levelDecoder.Decode(&_bitStream); + unsigned sym; + Z7_LZX_HUFF_DECODE_CHECK_NO(sym, &_levelDecoder, kNumTableBits_Level, MOVE_POS_CHECK) + // Z7_HUFF_DECODE_CHECK(sym, &_levelDecoder, kNumHuffmanBits, kNumTableBits_Level, &bitStream, return false) + // sym = _levelDecoder.Decode(&bitStream); + // if (!_levelDecoder.Decode_SymCheck_MovePosCheck(&bitStream, sym)) return false; + if (sym <= kNumHuffmanBits) { - int delta = (int)levels[i] - (int)sym; - delta += (delta < 0) ? (kNumHuffmanBits + 1) : 0; - levels[i++] = (Byte)delta; + int delta = (int)*levels - (int)sym; + delta += delta < 0 ? kNumHuffmanBits + 1 : 0; + *levels++ = (Byte)delta; continue; } unsigned num; - Byte symbol; + int symbol; if (sym < kLevelSym_Same) { - sym -= kLevelSym_Zero1; - num = kLevelSym_Zero1_Start + ((unsigned)sym << kLevelSym_Zero1_NumBits) + - (unsigned)ReadBits(kLevelSym_Zero1_NumBits + sym); + // sym -= kLevelSym_Zero1; + MACRO_ReadBits_NonZero(num, kLevelSym_Zero1_NumBits + (sym - kLevelSym_Zero1)) + num += (sym << kLevelSym_Zero1_NumBits) - (kLevelSym_Zero1 << kLevelSym_Zero1_NumBits) + kLevelSym_Zero1_Start; symbol = 0; } - else if (sym == kLevelSym_Same) + // else if (sym != kLevelSym_Same) return DECODE_ERROR_CODE; + else // (sym == kLevelSym_Same) { - num = kLevelSym_Same_Start + (unsigned)ReadBits(kLevelSym_Same_NumBits); - sym = _levelDecoder.Decode(&_bitStream); - if (sym > kNumHuffmanBits) - return false; - int delta = (int)levels[i] - (int)sym; - delta += (delta < 0) ? (kNumHuffmanBits + 1) : 0; - symbol = (Byte)delta; + MACRO_ReadBits_NonZero(num, kLevelSym_Same_NumBits) + num += kLevelSym_Same_Start; + // + (unsigned)bitStream.ReadBitsSmall(kLevelSym_Same_NumBits); + // Z7_HUFF_DECODE_CHECK(sym, &_levelDecoder, kNumHuffmanBits, kNumTableBits_Level, &bitStream, return DECODE_ERROR_CODE) + // if (!_levelDecoder.Decode2(&bitStream, sym)) return DECODE_ERROR_CODE; + // sym = _levelDecoder.Decode(&bitStream); + + Z7_LZX_HUFF_DECODE_CHECK_NO(sym, &_levelDecoder, kNumTableBits_Level, MOVE_POS) + + if (sym > kNumHuffmanBits) return DECODE_ERROR_CODE; + symbol = *levels - (int)sym; + symbol += symbol < 0 ? kNumHuffmanBits + 1 : 0; } - else - return false; - const unsigned limit = i + num; - if (limit > numSymbols) + if (num > (size_t)(levelsEnd - levels)) return false; - + const Byte *limit = levels + num; do - levels[i++] = symbol; - while (i < limit); + *levels++ = (Byte)symbol; + while (levels != limit); } - while (i < numSymbols); + while (levels != levelsEnd); - return true; + return _bitOffset; } -bool CDecoder::ReadTables(void) -{ - { - if (_skipByte) - { - if (_bitStream.DirectReadByte() != 0) - return false; - } +static const unsigned kPosSlotDelta = 256 / kNumLenSlots - kPosSlotOffset; + - _bitStream.NormalizeBig(); +#define READ_TABLE(_bitStream, levels, levelsEnd) \ +{ \ + _bitStream._bitOffset = ReadTable(_bitStream._bitOffset, _bitStream._buf, levels, levelsEnd); \ + if (_bitStream.IsOverRead()) return false; \ +} - const unsigned blockType = (unsigned)ReadBits(kBlockType_NumBits); - if (blockType > kBlockType_Uncompressed) +// can over-read input buffer for less than 32 bytes +bool CDecoder::ReadTables(CBitByteDecoder &_bitStream) throw() +{ + UPDATE_STAT(g_stats_NumTables++;) + { + const unsigned blockType = (unsigned)ReadBits(_bitStream, kBlockType_NumBits); + // if (blockType > kBlockType_Uncompressed || blockType == 0) + if ((unsigned)(blockType - 1) > kBlockType_Uncompressed - 1) return false; - - _unpackBlockSize = (1 << 15); - if (!_wimMode || ReadBits(1) == 0) + _unpackBlockSize = 1u << 15; + if (!_wimMode || ReadBits(_bitStream, 1) == 0) { - _unpackBlockSize = ReadBits(16); + _unpackBlockSize = ReadBits(_bitStream, 16); // wimlib supports chunks larger than 32KB (unsupported my MS wim). if (!_wimMode || _numDictBits >= 16) { _unpackBlockSize <<= 8; - _unpackBlockSize |= ReadBits(8); + _unpackBlockSize |= ReadBits(_bitStream, 8); } } PRF(printf("\nBlockSize = %6d %s ", _unpackBlockSize, (_pos & 1) ? "@@@" : " ")); _isUncompressedBlock = (blockType == kBlockType_Uncompressed); - _skipByte = false; if (_isUncompressedBlock) { _skipByte = ((_unpackBlockSize & 1) != 0); - - PRF(printf(" UncompressedBlock ")); - if (_unpackBlockSize & 1) - { - PRF(printf(" ######### ")); - } - - if (!_bitStream.PrepareUncompressed()) + // printf("\n UncompressedBlock %d", _unpackBlockSize); + PRF(printf(" UncompressedBlock ");) + // if (_unpackBlockSize & 1) { PRF(printf(" ######### ")); } + if (!_bitStream.Switch_To_ByteMode()) return false; if (_bitStream.GetRem() < kNumReps * 4) return false; - for (unsigned i = 0; i < kNumReps; i++) { const UInt32 rep = _bitStream.ReadUInt32(); - if (rep > _winSize) + // here we allow only such values for (rep) that can be set also by LZ code: + if (rep == 0 || rep > _winSize - kNumReps) return false; - _reps[i] = rep; + _reps[(size_t)i + kPosSlotOffset] = rep; } - + // printf("\n"); return true; } - - _numAlignBits = 64; - + + // _numAlignBits = 64; + // const UInt32 k_numAlignBits_PosSlots_MAX = 64 + kPosSlotDelta; + // _numAlignBits_PosSlots = k_numAlignBits_PosSlots_MAX; + const UInt32 k_numAlignBits_Dist_MAX = (UInt32)(Int32)-1; + _numAlignBits_Dist = k_numAlignBits_Dist_MAX; if (blockType == kBlockType_Aligned) { Byte levels[kAlignTableSize]; - _numAlignBits = kNumAlignBits; + // unsigned not0 = 0; + unsigned not3 = 0; for (unsigned i = 0; i < kAlignTableSize; i++) - levels[i] = (Byte)ReadBits(kNumAlignLevelBits); - RIF(_alignDecoder.Build(levels)) + { + const unsigned val = ReadBits(_bitStream, kNumAlignLevelBits); + levels[i] = (Byte)val; + // not0 |= val; + not3 |= (val ^ 3); + } + // static unsigned number = 0, all = 0; all++; + // if (!not0) return false; // Build(true) will test this case + if (not3) + { + // _numAlignBits_PosSlots = (kNumAlignBits + 1) * 2 + kPosSlotDelta; + // _numAlignBits = kNumAlignBits; + _numAlignBits_Dist = (1u << (kNumAlignBits + 1)) - (kNumReps - 1); + RIF(_alignDecoder.Build(levels, true)) // full + } + // else { number++; if (number % 4 == 0) printf("\nnumber= %u : %u%%", number, number * 100 / all); } + } + // if (_numAlignBits_PosSlots == k_numAlignBits_PosSlots_MAX) + if (_numAlignBits_Dist == k_numAlignBits_Dist_MAX) + { + size_t i; + for (i = 3; i < kNumLinearPosSlotBits; i++) + { + _extra[i * 2 + 2 + kPosSlotOffset] = (Byte)(SET_NUM_BITS(i)); + _extra[i * 2 + 3 + kPosSlotOffset] = (Byte)(SET_NUM_BITS(i)); + } + for (i = kNumLinearPosSlotBits * 2 + 2; i < kNumPosSlots; i++) + _extra[i + kPosSlotOffset] = (Byte)SET_NUM_BITS(kNumLinearPosSlotBits); + } + else + { + size_t i; + for (i = 3; i < kNumLinearPosSlotBits; i++) + { + _extra[i * 2 + 2 + kPosSlotOffset] = (Byte)(SET_NUM_BITS(i) - 3); + _extra[i * 2 + 3 + kPosSlotOffset] = (Byte)(SET_NUM_BITS(i) - 3); + } + for (i = kNumLinearPosSlotBits * 2 + 2; i < kNumPosSlots; i++) + _extra[i + kPosSlotOffset] = (Byte)(SET_NUM_BITS(kNumLinearPosSlotBits) - 3); } } - RIF(ReadTable(_mainLevels, 256)) - RIF(ReadTable(_mainLevels + 256, _numPosLenSlots)) - unsigned end = 256 + _numPosLenSlots; + READ_TABLE(_bitStream, _mainLevels, _mainLevels + 256) + READ_TABLE(_bitStream, _mainLevels + 256, _mainLevels + 256 + _numPosLenSlots) + const unsigned end = 256 + _numPosLenSlots; memset(_mainLevels + end, 0, kMainTableSize - end); - RIF(_mainDecoder.Build(_mainLevels)) - RIF(ReadTable(_lenLevels, kNumLenSymbols)) - return _lenDecoder.Build(_lenLevels); + // #define NUM_CYC 1 + // unsigned j; for (j = 0; j < NUM_CYC; j++) + RIF(_mainDecoder.Build(_mainLevels, NHuffman::k_BuildMode_Full)) + // if (kNumLenSymols_Big_Start) + memset(_lenLevels, 0, kNumLenSymols_Big_Start); + READ_TABLE(_bitStream, + _lenLevels + kNumLenSymols_Big_Start, + _lenLevels + kNumLenSymols_Big_Start + kNumLenSymbols) + // for (j = 0; j < NUM_CYC; j++) + RIF(_lenDecoder.Build(_lenLevels, NHuffman::k_BuildMode_Full_or_Empty)) + return true; +} + + + +static ptrdiff_t CodeLz(CDecoder *dec, size_t next, ptrdiff_t _bitOffset, const Byte *_buf) throw() +{ + { + Byte *const win = dec->_win; + const UInt32 winSize = dec->_winSize; + Byte *pos = win + dec->_pos; + const Byte * const posEnd = pos + next; + NHuffman::CValueInt _value; + + NORMALIZE + +#if 1 + #define HUFF_DEC_PREFIX dec-> +#else + const NHuffman::CDecoder _mainDecoder = dec->_mainDecoder; + const NHuffman::CDecoder256 _lenDecoder = dec->_lenDecoder; + const NHuffman::CDecoder7b _alignDecoder = dec->_alignDecoder; + #define HUFF_DEC_PREFIX +#endif + + do + { + unsigned sym; + // printf("\npos = %6u", pos - win); + { + const NHuffman::CDecoder + *mainDecoder = & HUFF_DEC_PREFIX _mainDecoder; + Z7_LZX_HUFF_DECODE_CHECK_NO(sym, mainDecoder, kNumTableBits_Main, MOVE_POS_CHECK_STAT) + } + // if (!_mainDecoder.Decode_SymCheck_MovePosCheck(&bitStream, sym)) return DECODE_ERROR_CODE; + // sym = _mainDecoder.Decode(&bitStream); + // if (bitStream.WasExtraReadError_Fast()) return DECODE_ERROR_CODE; + + // printf(" sym = %3x", sym); + UPDATE_STAT(g_stats_main[sym]++;) + + if (sym < 256) + { + UPDATE_STAT(g_stats_NumLits++;) + *pos++ = (Byte)sym; + } + else + { + // sym -= 256; + // if (sym >= _numPosLenSlots) return DECODE_ERROR_CODE; + const unsigned posSlot = sym / kNumLenSlots; + unsigned len = sym % kNumLenSlots + kMatchMinLen; + if (len == kNumLenSlots - 1 + kMatchMinLen) + { + const NHuffman::CDecoder256 + *lenDecoder = & HUFF_DEC_PREFIX _lenDecoder; + Z7_LZX_HUFF_DECODE_CHECK_YES(len, lenDecoder, kNumTableBits_Len, MOVE_POS_STAT) + // if (!_lenDecoder.Decode2(&bitStream, len)) return DECODE_ERROR_CODE; + // len = _lenDecoder.Decode(&bitStream); + // if (len >= kNumLenSymbols) return DECODE_ERROR_CODE; + UPDATE_STAT(g_stats_len[len - kNumLenSymols_Big_Start]++;) + len += kNumLenSlots - 1 + kMatchMinLen - kNumLenSymols_Big_Start; + } + /* + if ((next -= len) < 0) + return DECODE_ERROR_CODE; + */ + UInt32 dist; + + dist = dec->_reps[(size_t)posSlot - kPosSlotDelta]; + if (posSlot < kNumReps + 256 / kNumLenSlots) + { + // if (posSlot != kNumReps + kPosSlotDelta) + // if (posSlot - (kNumReps + kPosSlotDelta + 1) < 2) + dec->_reps[(size_t)posSlot - kPosSlotDelta] = dec->_reps[kPosSlotOffset]; + /* + if (posSlot != kPosSlotDelta) + { + UInt32 temp = dist; + if (posSlot == kPosSlotDelta + 1) + { + dist = reps[1]; + reps[1] = temp; + } + else + { + dist = reps[2]; + reps[2] = temp; + } + // dist = reps[(size_t)(posSlot) - kPosSlotDelta]; + // reps[(size_t)(posSlot) - kPosSlotDelta] = reps[0]; + // reps[(size_t)(posSlot) - kPosSlotDelta] = temp; + } + */ + } + else // if (posSlot != kNumReps + kPosSlotDelta) + { + unsigned numDirectBits; +#if 0 + if (posSlot < kNumPowerPosSlots + kPosSlotDelta) + { + numDirectBits = (posSlot - 2 - kPosSlotDelta) >> 1; + dist = (UInt32)(2 | (posSlot & 1)) << numDirectBits; + } + else + { + numDirectBits = kNumLinearPosSlotBits; + dist = (UInt32)(posSlot - 0x22 - kPosSlotDelta) << kNumLinearPosSlotBits; + } + dist -= kNumReps - 1; +#else + numDirectBits = dec->_extra[(size_t)posSlot - kPosSlotDelta]; + // dist = reps[(size_t)(posSlot) - kPosSlotDelta]; +#endif + dec->_reps[kPosSlotOffset + 2] = + dec->_reps[kPosSlotOffset + 1]; + dec->_reps[kPosSlotOffset + 1] = + dec->_reps[kPosSlotOffset + 0]; + + // dist += val; dist += bitStream.ReadBitsBig(numDirectBits); + // if (posSlot >= _numAlignBits_PosSlots) + // if (numDirectBits >= _numAlignBits) + // if (val >= _numAlignBits_Dist) + // UInt32 val; MACRO_ReadBitsBig(val , numDirectBits) + // dist += val; + // dist += (UInt32)((UInt32)_value >> 1 >> (/* 31 ^ */ (numDirectBits))); + // MOVE_POS((numDirectBits ^ 31)) + MACRO_ReadBitsBig_pre(numDirectBits) + // dist += (UInt32)_value; + if (dist >= dec->_numAlignBits_Dist) + { + // if (numDirectBits != _numAlignBits) + { + // UInt32 val; + // dist -= (UInt32)_value; + MACRO_ReadBitsBig_add3(dist) + NORMALIZE + // dist += (val << kNumAlignBits); + // dist += bitStream.ReadBitsSmall(numDirectBits - kNumAlignBits) << kNumAlignBits; + } + { + // const unsigned alignTemp = _alignDecoder.Decode(&bitStream); + const NHuffman::CDecoder7b *alignDecoder = & HUFF_DEC_PREFIX _alignDecoder; + unsigned alignTemp; + UPDATE_STAT(g_stats_NumAlign++;) + Z7_HUFF_DECODER_7B_DECODE(alignTemp, alignDecoder, GET_VAL_BASE, MOVE_POS, bs) + // NORMALIZE + // if (alignTemp >= kAlignTableSize) return DECODE_ERROR_CODE; + dist += alignTemp; + } + } + else + { + { + MACRO_ReadBitsBig_add(dist) + // dist += bitStream.ReadBitsSmall(numDirectBits - kNumAlignBits) << kNumAlignBits; + } + } + NORMALIZE + /* + else + { + UInt32 val; + MACRO_ReadBitsBig(val, numDirectBits) + dist += val; + // dist += bitStream.ReadBitsBig(numDirectBits); + } + */ + } + dec->_reps[kPosSlotOffset + 0] = dist; + + Byte *dest = pos; + if (len > (size_t)(posEnd - pos)) + return DECODE_ERROR_CODE; + Int32 srcPos = (Int32)(pos - win); + pos += len; + srcPos -= (Int32)dist; + if (srcPos < 0) // fast version + { + if (!dec->_overDict) + return DECODE_ERROR_CODE; + srcPos &= winSize - 1; + UInt32 rem = winSize - (UInt32)srcPos; + if (len > rem) + { + len -= rem; + const Byte *src = win + (UInt32)srcPos; + do + *dest++ = *src++; + while (--rem); + srcPos = 0; + } + } + CopyLzMatch(dest, win + (UInt32)srcPos, len, dist); + } + } + while (pos != posEnd); + + return _bitOffset; + } } -HRESULT CDecoder::CodeSpec(UInt32 curSize) + + +// inSize != 0 +// outSize != 0 ??? +HRESULT CDecoder::CodeSpec(const Byte *inData, size_t inSize, UInt32 outSize) throw() { - if (!_keepHistory || !_isUncompressedBlock) - _bitStream.NormalizeBig(); + // ((inSize & 1) != 0) case is possible, if current call will be finished with Uncompressed Block. + CBitByteDecoder _bitStream; + if (_keepHistory && _isUncompressedBlock) + _bitStream.Init_ByteMode(inData, inSize); + else + _bitStream.Init_BitMode(inData, inSize); if (!_keepHistory) { + _isUncompressedBlock = false; _skipByte = false; _unpackBlockSize = 0; - - memset(_mainLevels, 0, kMainTableSize); - memset(_lenLevels, 0, kNumLenSymbols); - + memset(_mainLevels, 0, sizeof(_mainLevels)); + memset(_lenLevels, 0, sizeof(_lenLevels)); { _x86_translationSize = 12000000; if (!_wimMode) { _x86_translationSize = 0; - if (ReadBits(1) != 0) + if (ReadBits(_bitStream, 1) != 0) { - UInt32 v = ReadBits(16) << 16; - v |= ReadBits(16); + UInt32 v = ReadBits(_bitStream, 16) << 16; + v |= ReadBits(_bitStream, 16); _x86_translationSize = v; } } - _x86_processedSize = 0; } - - _reps[0] = 1; - _reps[1] = 1; - _reps[2] = 1; + _reps[0 + kPosSlotOffset] = 1; + _reps[1 + kPosSlotOffset] = 1; + _reps[2 + kPosSlotOffset] = 1; } - while (curSize > 0) + while (outSize) { + /* + // check it for bit mode only: if (_bitStream.WasExtraReadError_Fast()) return S_FALSE; - + */ if (_unpackBlockSize == 0) { - if (!ReadTables()) + if (_skipByte) + { + if (_bitStream.GetRem() < 1) + return S_FALSE; + if (_bitStream.DirectReadByte() != 0) + return S_FALSE; + } + if (_isUncompressedBlock) + _bitStream.Switch_To_BitMode(); + if (!ReadTables(_bitStream)) return S_FALSE; continue; } + // _unpackBlockSize != 0 UInt32 next = _unpackBlockSize; - if (next > curSize) - next = curSize; + if (next > outSize) + next = outSize; + // next != 0 + + // PRF(printf("\nnext = %d", (unsigned)next);) if (_isUncompressedBlock) { - const size_t rem = _bitStream.GetRem(); - if (rem == 0) + if (_bitStream.GetRem() < next) return S_FALSE; - if (next > rem) - next = (UInt32)rem; _bitStream.CopyTo(_win + _pos, next); _pos += next; - curSize -= next; _unpackBlockSize -= next; - - /* we don't know where skipByte can be placed, if it's end of chunk: - 1) in current chunk - there are such cab archives, if chunk is last - 2) in next chunk - are there such archives ? */ - - if (_skipByte - && _unpackBlockSize == 0 - && curSize == 0 - && _bitStream.IsOneDirectByteLeft()) - { - _skipByte = false; - if (_bitStream.DirectReadByte() != 0) - return S_FALSE; - } - - continue; } - - curSize -= next; - _unpackBlockSize -= next; - - Byte *win = _win; - - while (next > 0) + else { - if (_bitStream.WasExtraReadError_Fast()) + _unpackBlockSize -= next; + _bitStream._bitOffset = CodeLz(this, next, _bitStream._bitOffset, _bitStream._buf); + if (_bitStream.IsOverRead()) return S_FALSE; + _pos += next; + } + outSize -= next; + } - UInt32 sym = _mainDecoder.Decode(&_bitStream); - - if (sym < 256) - { - win[_pos++] = (Byte)sym; - next--; - continue; - } - { - sym -= 256; - if (sym >= _numPosLenSlots) - return S_FALSE; - const UInt32 posSlot = sym / kNumLenSlots; - const UInt32 lenSlot = sym % kNumLenSlots; - UInt32 len = kMatchMinLen + lenSlot; - - if (lenSlot == kNumLenSlots - 1) - { - UInt32 lenTemp = _lenDecoder.Decode(&_bitStream); - if (lenTemp >= kNumLenSymbols) - return S_FALSE; - len = kMatchMinLen + kNumLenSlots - 1 + lenTemp; - } - - UInt32 dist; - - if (posSlot < kNumReps) - { - dist = _reps[posSlot]; - _reps[posSlot] = _reps[0]; - _reps[0] = dist; - } - else - { - unsigned numDirectBits; - - if (posSlot < kNumPowerPosSlots) - { - numDirectBits = (unsigned)(posSlot >> 1) - 1; - dist = ((2 | (posSlot & 1)) << numDirectBits); - } - else - { - numDirectBits = kNumLinearPosSlotBits; - dist = ((posSlot - 0x22) << kNumLinearPosSlotBits); - } - - if (numDirectBits >= _numAlignBits) - { - dist += (_bitStream.ReadBitsSmall(numDirectBits - kNumAlignBits) << kNumAlignBits); - const UInt32 alignTemp = _alignDecoder.Decode(&_bitStream); - if (alignTemp >= kAlignTableSize) - return S_FALSE; - dist += alignTemp; - } - else - dist += _bitStream.ReadBitsBig(numDirectBits); - - dist -= kNumReps - 1; - _reps[2] = _reps[1]; - _reps[1] = _reps[0]; - _reps[0] = dist; - } - - if (len > next) - return S_FALSE; - - if (dist > _pos && !_overDict) - return S_FALSE; - - Byte *dest = win + _pos; - const UInt32 mask = (_winSize - 1); - UInt32 srcPos = (_pos - dist) & mask; + // outSize == 0 - next -= len; - - if (len > _winSize - srcPos) - { - _pos += len; - do - { - *dest++ = win[srcPos++]; - srcPos &= mask; - } - while (--len); - } - else - { - const ptrdiff_t src = (ptrdiff_t)srcPos - (ptrdiff_t)_pos; - _pos += len; - const Byte *lim = dest + len; - *(dest) = *(dest + src); - dest++; - do - *(dest) = *(dest + src); - while (++dest != lim); - } - } + if (_isUncompressedBlock) + { + /* we don't know where skipByte can be placed, if it's end of chunk: + 1) in current chunk - there are such cab archives, if chunk is last + 2) in next chunk - are there such archives ? */ + if (_unpackBlockSize == 0 + && _skipByte + // && outSize == 0 + && _bitStream.IsOneDirectByteLeft()) + { + _skipByte = false; + if (_bitStream.DirectReadByte() != 0) + return S_FALSE; } } - if (!_bitStream.WasFinishedOK()) + if (_bitStream.GetRem() != 0) return S_FALSE; - + if (!_isUncompressedBlock) + if (!_bitStream.WasBitStreamFinishedOK()) + return S_FALSE; return S_OK; } -HRESULT CDecoder::Code(const Byte *inData, size_t inSize, UInt32 outSize) +#if k_Filter_OutBufSize_Add > k_Lz_OutBufSize_Add + #define k_OutBufSize_Add k_Filter_OutBufSize_Add +#else + #define k_OutBufSize_Add k_Lz_OutBufSize_Add +#endif + +HRESULT CDecoder::Code_WithExceedReadWrite(const Byte *inData, size_t inSize, UInt32 outSize) throw() { if (!_keepHistory) { @@ -466,63 +1452,65 @@ HRESULT CDecoder::Code(const Byte *inData, size_t inSize, UInt32 outSize) { _pos = 0; _overDict = true; +#if k_OutBufSize_Add > 0 + // data after (_winSize) can be used, because we can use overwrite. + // memset(_win + _winSize, 0, k_OutBufSize_Add); +#endif } - _writePos = _pos; _unpackedData = _win + _pos; - + if (outSize > _winSize - _pos) return S_FALSE; + + PRF(printf("\ninSize = %d", (unsigned)inSize);) + PRF(if ((inSize & 1) != 0) printf("---------");) - PRF(printf("\ninSize = %d", inSize)); - if ((inSize & 1) != 0) - { - PRF(printf(" ---------")); - } - - if (inSize < 1) + if (inSize == 0) return S_FALSE; - - _bitStream.Init(inData, inSize); - - const HRESULT res = CodeSpec(outSize); + const HRESULT res = CodeSpec(inData, inSize, outSize); const HRESULT res2 = Flush(); return (res == S_OK ? res2 : res); } -HRESULT CDecoder::SetParams2(unsigned numDictBits) +HRESULT CDecoder::SetParams2(unsigned numDictBits) throw() { - _numDictBits = numDictBits; - if (numDictBits < kNumDictBits_Min || numDictBits > kNumDictBits_Max) + if (numDictBits < kNumDictBits_Min || + numDictBits > kNumDictBits_Max) return E_INVALIDARG; - const unsigned numPosSlots = (numDictBits < 20) ? - numDictBits * 2 : - 34 + ((unsigned)1 << (numDictBits - 17)); - _numPosLenSlots = numPosSlots * kNumLenSlots; + _numDictBits = (Byte)numDictBits; + const unsigned numPosSlots2 = (numDictBits < 20) ? + numDictBits : 17 + (1u << (numDictBits - 18)); + _numPosLenSlots = numPosSlots2 * (kNumLenSlots * 2); return S_OK; } -HRESULT CDecoder::SetParams_and_Alloc(unsigned numDictBits) +HRESULT CDecoder::Set_DictBits_and_Alloc(unsigned numDictBits) throw() { RINOK(SetParams2(numDictBits)) - const UInt32 newWinSize = (UInt32)1 << numDictBits; - - if (NeedAlloc) + if (_needAlloc) { if (!_win || newWinSize != _winSize) { - ::MidFree(_win); + // BigFree + z7_AlignedFree + (_win); _winSize = 0; - _win = (Byte *)::MidAlloc(newWinSize); + const size_t alloc_size = newWinSize + k_OutBufSize_Add; + _win = (Byte *) + // BigAlloc + z7_AlignedAlloc + (alloc_size); if (!_win) return E_OUTOFMEMORY; + // optional: + memset(_win, 0, alloc_size); } } - - _winSize = (UInt32)newWinSize; + _winSize = newWinSize; return S_OK; } diff --git a/CPP/7zip/Compress/LzxDecoder.h b/CPP/7zip/Compress/LzxDecoder.h index ab114e5..b4556a6 100644 --- a/CPP/7zip/Compress/LzxDecoder.h +++ b/CPP/7zip/Compress/LzxDecoder.h @@ -3,239 +3,102 @@ #ifndef ZIP7_INC_LZX_DECODER_H #define ZIP7_INC_LZX_DECODER_H -#include "../../../C/CpuArch.h" - -#include "../../Common/MyCom.h" - #include "HuffmanDecoder.h" #include "Lzx.h" namespace NCompress { namespace NLzx { -class CBitDecoder -{ - unsigned _bitPos; - UInt32 _value; - const Byte *_buf; - const Byte *_bufLim; - UInt32 _extraSize; -public: - - void Init(const Byte *data, size_t size) - { - _buf = data; - _bufLim = data + size - 1; - _bitPos = 0; - _extraSize = 0; - } - - size_t GetRem() const { return (size_t)(_bufLim + 1 - _buf); } - bool WasExtraReadError_Fast() const { return _extraSize > 4; } - - bool WasFinishedOK() const - { - if (_buf != _bufLim + 1) - return false; - if ((_bitPos >> 4) * 2 != _extraSize) - return false; - unsigned numBits = _bitPos & 15; - return (((_value >> (_bitPos - numBits)) & (((UInt32)1 << numBits) - 1)) == 0); - } - - void NormalizeSmall() - { - if (_bitPos <= 16) - { - UInt32 val; - if (_buf >= _bufLim) - { - val = 0xFFFF; - _extraSize += 2; - } - else - { - val = GetUi16(_buf); - _buf += 2; - } - _value = (_value << 16) | val; - _bitPos += 16; - } - } - - void NormalizeBig() - { - if (_bitPos <= 16) - { - { - UInt32 val; - if (_buf >= _bufLim) - { - val = 0xFFFF; - _extraSize += 2; - } - else - { - val = GetUi16(_buf); - _buf += 2; - } - _value = (_value << 16) | val; - _bitPos += 16; - } - if (_bitPos <= 16) - { - UInt32 val; - if (_buf >= _bufLim) - { - val = 0xFFFF; - _extraSize += 2; - } - else - { - val = GetUi16(_buf); - _buf += 2; - } - _value = (_value << 16) | val; - _bitPos += 16; - } - } - } - - UInt32 GetValue(unsigned numBits) const - { - return (_value >> (_bitPos - numBits)) & (((UInt32)1 << numBits) - 1); - } - - void MovePos(unsigned numBits) - { - _bitPos -= numBits; - NormalizeSmall(); - } - - UInt32 ReadBitsSmall(unsigned numBits) - { - _bitPos -= numBits; - UInt32 val = (_value >> _bitPos) & (((UInt32)1 << numBits) - 1); - NormalizeSmall(); - return val; - } +const unsigned kAdditionalOutputBufSize = 32 * 2; - UInt32 ReadBitsBig(unsigned numBits) - { - _bitPos -= numBits; - UInt32 val = (_value >> _bitPos) & (((UInt32)1 << numBits) - 1); - NormalizeBig(); - return val; - } +const unsigned kNumTableBits_Main = 11; +const unsigned kNumTableBits_Len = 8; - bool PrepareUncompressed() - { - if (_extraSize != 0) - return false; - unsigned numBits = _bitPos - 16; - if (((_value >> 16) & (((UInt32)1 << numBits) - 1)) != 0) - return false; - _buf -= 2; - _bitPos = 0; - return true; - } +// if (kNumLenSymols_Big <= 256) we can use NHuffman::CDecoder256 +// if (kNumLenSymols_Big > 256) we must use NHuffman::CDecoder +// const unsigned kNumLenSymols_Big_Start = kNumLenSlots - 1 + kMatchMinLen; // 8 - 1 + 2 +const unsigned kNumLenSymols_Big_Start = 0; +// const unsigned kNumLenSymols_Big_Start = 0; +const unsigned kNumLenSymols_Big = kNumLenSymols_Big_Start + kNumLenSymbols; - UInt32 ReadUInt32() - { - UInt32 v = GetUi32(_buf); - _buf += 4; - return v; - } - - void CopyTo(Byte *dest, size_t size) - { - memcpy(dest, _buf, size); - _buf += size; - } - - bool IsOneDirectByteLeft() const { return _buf == _bufLim && _extraSize == 0; } - - Byte DirectReadByte() - { - if (_buf > _bufLim) - { - _extraSize++; - return 0xFF; - } - return *_buf++; - } -}; +#if 1 + // for smallest structure size: + const unsigned kPosSlotOffset = 0; +#else + // use virtual entries for mispredicted branches: + const unsigned kPosSlotOffset = 256 / kNumLenSlots; +#endif +class CBitByteDecoder; -Z7_CLASS_IMP_COM_0( - CDecoder -) - CBitDecoder _bitStream; - Byte *_win; +class CDecoder +{ +public: UInt32 _pos; UInt32 _winSize; + Byte *_win; bool _overDict; bool _isUncompressedBlock; bool _skipByte; - unsigned _numAlignBits; + bool _keepHistory; + bool _keepHistoryForNext; + bool _needAlloc; + bool _wimMode; + Byte _numDictBits; - UInt32 _reps[kNumReps]; - UInt32 _numPosLenSlots; + // unsigned _numAlignBits_PosSlots; + // unsigned _numAlignBits; + UInt32 _numAlignBits_Dist; +private: + unsigned _numPosLenSlots; UInt32 _unpackBlockSize; -public: - bool KeepHistoryForNext; - bool NeedAlloc; -private: - bool _keepHistory; - bool _wimMode; - unsigned _numDictBits; UInt32 _writePos; - Byte *_x86_buf; UInt32 _x86_translationSize; UInt32 _x86_processedSize; + Byte *_x86_buf; Byte *_unpackedData; - - NHuffman::CDecoder _mainDecoder; - NHuffman::CDecoder _lenDecoder; - NHuffman::CDecoder7b _alignDecoder; - NHuffman::CDecoder _levelDecoder; +public: + Byte _extra[kPosSlotOffset + kNumPosSlots]; + UInt32 _reps[kPosSlotOffset + kNumPosSlots]; + NHuffman::CDecoder _mainDecoder; + NHuffman::CDecoder256 _lenDecoder; + NHuffman::CDecoder7b _alignDecoder; +private: Byte _mainLevels[kMainTableSize]; - Byte _lenLevels[kNumLenSymbols]; - - HRESULT Flush(); + Byte _lenLevels[kNumLenSymols_Big]; - UInt32 ReadBits(unsigned numBits); - bool ReadTable(Byte *levels, unsigned numSymbols); - bool ReadTables(); + HRESULT Flush() throw(); + bool ReadTables(CBitByteDecoder &_bitStream) throw(); - HRESULT CodeSpec(UInt32 size); - HRESULT SetParams2(unsigned numDictBits); + HRESULT CodeSpec(const Byte *inData, size_t inSize, UInt32 outSize) throw(); + HRESULT SetParams2(unsigned numDictBits) throw(); public: - CDecoder(bool wimMode = false); - ~CDecoder(); + CDecoder() throw(); + ~CDecoder() throw(); + + void Set_WimMode(bool wimMode) { _wimMode = wimMode; } + void Set_KeepHistory(bool keepHistory) { _keepHistory = keepHistory; } + void Set_KeepHistoryForNext(bool keepHistoryForNext) { _keepHistoryForNext = keepHistoryForNext; } - HRESULT SetExternalWindow(Byte *win, unsigned numDictBits) + HRESULT Set_ExternalWindow_DictBits(Byte *win, unsigned numDictBits) { - NeedAlloc = false; + _needAlloc = false; _win = win; _winSize = (UInt32)1 << numDictBits; return SetParams2(numDictBits); } + HRESULT Set_DictBits_and_Alloc(unsigned numDictBits) throw(); - void SetKeepHistory(bool keepHistory) { _keepHistory = keepHistory; } - - HRESULT SetParams_and_Alloc(unsigned numDictBits); - - HRESULT Code(const Byte *inData, size_t inSize, UInt32 outSize); + HRESULT Code_WithExceedReadWrite(const Byte *inData, size_t inSize, UInt32 outSize) throw(); - bool WasBlockFinished() const { return _unpackBlockSize == 0; } + bool WasBlockFinished() const { return _unpackBlockSize == 0; } const Byte *GetUnpackData() const { return _unpackedData; } - UInt32 GetUnpackSize() const { return _pos - _writePos; } + UInt32 GetUnpackSize() const { return _pos - _writePos; } }; }} diff --git a/CPP/7zip/Compress/PpmdDecoder.cpp b/CPP/7zip/Compress/PpmdDecoder.cpp index 1238df6..3dbde75 100644 --- a/CPP/7zip/Compress/PpmdDecoder.cpp +++ b/CPP/7zip/Compress/PpmdDecoder.cpp @@ -74,6 +74,7 @@ HRESULT CDecoder::CodeSpec(Byte *memStream, UInt32 size) _status = kStatus_Normal; Ppmd7_Init(&_ppmd, _order); break; + default: break; } if (_outSizeDefined) diff --git a/CPP/7zip/Compress/PpmdZip.cpp b/CPP/7zip/Compress/PpmdZip.cpp index 5039131..e59e52f 100644 --- a/CPP/7zip/Compress/PpmdZip.cpp +++ b/CPP/7zip/Compress/PpmdZip.cpp @@ -12,6 +12,15 @@ namespace NCompress { namespace NPpmdZip { +static const UInt32 kBufSize = 1 << 20; + +bool CBuf::Alloc() +{ + if (!Buf) + Buf = (Byte *)::MidAlloc(kBufSize); + return (Buf != NULL); +} + CDecoder::CDecoder(bool fullFileMode): _fullFileMode(fullFileMode) { @@ -255,7 +264,10 @@ Z7_COM7F_IMF(CEncoder::Code(ISequentialInStream *inStream, ISequentialOutStream Ppmd8_Init(&_ppmd, (unsigned)_props.Order, (unsigned)_props.Restor); { - const UInt32 val = (UInt32)(((unsigned)_props.Order - 1) + ((_props.MemSizeMB - 1) << 4) + ((unsigned)_props.Restor << 12)); + const unsigned val = + ((unsigned)_props.Order - 1) + + (((unsigned)_props.MemSizeMB - 1) << 4) + + ((unsigned)_props.Restor << 12); _outStream.WriteByte((Byte)(val & 0xFF)); _outStream.WriteByte((Byte)(val >> 8)); } @@ -294,7 +306,4 @@ Z7_COM7F_IMF(CEncoder::Code(ISequentialInStream *inStream, ISequentialOutStream } } - - - }} diff --git a/CPP/7zip/Compress/PpmdZip.h b/CPP/7zip/Compress/PpmdZip.h index a23d008..6cc6af1 100644 --- a/CPP/7zip/Compress/PpmdZip.h +++ b/CPP/7zip/Compress/PpmdZip.h @@ -15,20 +15,13 @@ namespace NCompress { namespace NPpmdZip { -static const UInt32 kBufSize = (1 << 20); - struct CBuf { Byte *Buf; CBuf(): Buf(NULL) {} ~CBuf() { ::MidFree(Buf); } - bool Alloc() - { - if (!Buf) - Buf = (Byte *)::MidAlloc(kBufSize); - return (Buf != NULL); - } + bool Alloc(); }; @@ -38,10 +31,10 @@ Z7_CLASS_IMP_NOQIB_3( , ICompressSetFinishMode , ICompressGetInStreamProcessedSize ) + bool _fullFileMode; CByteInBufWrap _inStream; CBuf _outStream; CPpmd8 _ppmd; - bool _fullFileMode; public: CDecoder(bool fullFileMode = true); ~CDecoder(); diff --git a/CPP/7zip/Compress/QuantumDecoder.cpp b/CPP/7zip/Compress/QuantumDecoder.cpp index 16ef0fc..98817a4 100644 --- a/CPP/7zip/Compress/QuantumDecoder.cpp +++ b/CPP/7zip/Compress/QuantumDecoder.cpp @@ -2,6 +2,11 @@ #include "StdAfx.h" +// #include + +#include "../../../C/Alloc.h" +#include "../../../C/CpuArch.h" + #include "../../Common/Defs.h" #include "QuantumDecoder.h" @@ -11,184 +16,339 @@ namespace NQuantum { static const unsigned kNumLenSymbols = 27; static const unsigned kMatchMinLen = 3; -static const unsigned kNumSimplePosSlots = 4; static const unsigned kNumSimpleLenSlots = 6; -static const UInt16 kUpdateStep = 8; -static const UInt16 kFreqSumMax = 3800; -static const unsigned kReorderCountStart = 4; +static const unsigned kUpdateStep = 8; +static const unsigned kFreqSumMax = 3800; +static const unsigned kReorderCount_Start = 4; static const unsigned kReorderCount = 50; -void CModelDecoder::Init(unsigned numItems) + +class CRangeDecoder { - NumItems = numItems; - ReorderCount = kReorderCountStart; - for (unsigned i = 0; i < numItems; i++) + UInt32 Low; + UInt32 Range; + UInt32 Code; + + unsigned _bitOffset; + const Byte *_buf; + const Byte *_bufLim; + +public: + + Z7_FORCE_INLINE + void Init(const Byte *inData, size_t inSize) { - Freqs[i] = (UInt16)(numItems - i); - Vals[i] = (Byte)i; + Code = ((UInt32)*inData << 8) | inData[1]; + _buf = inData + 2; + _bufLim = inData + inSize; + _bitOffset = 0; + Low = 0; + Range = 0x10000; } - Freqs[numItems] = 0; -} + Z7_FORCE_INLINE + bool WasExtraRead() const + { + return _buf > _bufLim; + } + + Z7_FORCE_INLINE + UInt32 ReadBits(unsigned numBits) // numBits > 0 + { + unsigned bitOffset = _bitOffset; + const Byte *buf = _buf; + const UInt32 res = GetBe32(buf) << bitOffset; + bitOffset += numBits; + _buf = buf + (bitOffset >> 3); + _bitOffset = bitOffset & 7; + return res >> (32 - numBits); + } + + // ---------- Range Decoder functions ---------- + + Z7_FORCE_INLINE + bool Finish() + { + const unsigned numBits = 2 + ((16 - 2 - _bitOffset) & 7); + if (ReadBits(numBits) != 0) + return false; + return _buf == _bufLim; + } + + Z7_FORCE_INLINE + UInt32 GetThreshold(UInt32 total) const + { + return ((Code + 1) * total - 1) / Range; // & 0xFFFF is not required; + } + + Z7_FORCE_INLINE + void Decode(UInt32 start, UInt32 end, UInt32 total) + { + // UInt32 hi = ~(Low + end * Range / total - 1); + UInt32 hi = 0 - (Low + end * Range / total); + const UInt32 offset = start * Range / total; + UInt32 lo = Low + offset; + Code -= offset; + UInt32 numBits = 0; + lo ^= hi; + while (lo & (1u << 15)) + { + lo <<= 1; + hi <<= 1; + numBits++; + } + lo ^= hi; + UInt32 an = lo & hi; + while (an & (1u << 14)) + { + an <<= 1; + lo <<= 1; + hi <<= 1; + numBits++; + } + Low = lo; + Range = ((~hi - lo) & 0xffff) + 1; + if (numBits) + Code = (Code << numBits) + ReadBits(numBits); + } +}; + + +// Z7_FORCE_INLINE +Z7_NO_INLINE unsigned CModelDecoder::Decode(CRangeDecoder *rc) +// Z7_NO_INLINE void CModelDecoder::Normalize() { - const UInt32 threshold = rc->GetThreshold(Freqs[0]); - unsigned i; - for (i = 1; Freqs[i] > threshold; i++); - - rc->Decode(Freqs[i], Freqs[(size_t)i - 1], Freqs[0]); - const unsigned res = Vals[--i]; - - do - Freqs[i] = (UInt16)(Freqs[i] + kUpdateStep); - while (i--); - if (Freqs[0] > kFreqSumMax) { if (--ReorderCount == 0) { ReorderCount = kReorderCount; - for (i = 0; i < NumItems; i++) - Freqs[i] = (UInt16)(((Freqs[i] - Freqs[(size_t)i + 1]) + 1) >> 1); - for (i = 0; i < NumItems - 1; i++) - for (unsigned j = i + 1; j < NumItems; j++) - if (Freqs[i] < Freqs[j]) - { - const UInt16 tmpFreq = Freqs[i]; - const Byte tmpVal = Vals[i]; - Freqs[i] = Freqs[j]; - Vals[i] = Vals[j]; - Freqs[j] = tmpFreq; - Vals[j] = tmpVal; - } - + { + unsigned i = NumItems; + unsigned next = 0; + UInt16 *freqs = &Freqs[i]; + do + { + const unsigned freq = *--freqs; + *freqs = (UInt16)((freq - next + 1) >> 1); + next = freq; + } + while (--i); + } + { + for (unsigned i = 0; i < NumItems - 1; i++) + { + UInt16 freq = Freqs[i]; + for (unsigned k = i + 1; k < NumItems; k++) + if (freq < Freqs[k]) + { + const UInt16 freq2 = Freqs[k]; + Freqs[k] = freq; + Freqs[i] = freq2; + freq = freq2; + const Byte val = Vals[i]; + Vals[i] = Vals[k]; + Vals[k] = val; + } + } + } + unsigned i = NumItems; + unsigned freq = 0; + UInt16 *freqs = &Freqs[i]; do - Freqs[i] = (UInt16)(Freqs[i] + Freqs[(size_t)i + 1]); - while (i--); + { + freq += *--freqs; + *freqs = (UInt16)freq; + } + while (--i); } else { - i = NumItems - 1; + unsigned i = NumItems; + unsigned next = 1; + UInt16 *freqs = &Freqs[i]; do { - Freqs[i] = (UInt16)(Freqs[i] >> 1); - if (Freqs[i] <= Freqs[(size_t)i + 1]) - Freqs[i] = (UInt16)(Freqs[(size_t)i + 1] + 1); + unsigned freq = *--freqs >> 1; + if (freq < next) + freq = next; + *freqs = (UInt16)freq; + next = freq + 1; } - while (i--); + while (--i); } } - + unsigned res; + { + const unsigned freq0 = Freqs[0]; + Freqs[0] = (UInt16)(freq0 + kUpdateStep); + const unsigned threshold = rc->GetThreshold(freq0); + UInt16 *freqs = &Freqs[1]; + unsigned freq = *freqs; + while (freq > threshold) + { + *freqs++ = (UInt16)(freq + kUpdateStep); + freq = *freqs; + } + res = Vals[freqs - Freqs - 1]; + rc->Decode(freq, freqs[-1] - kUpdateStep, freq0); + } return res; } -void CDecoder::Init() +Z7_NO_INLINE +void CModelDecoder::Init(unsigned numItems, unsigned startVal) { - m_Selector.Init(kNumSelectors); - unsigned i; - for (i = 0; i < kNumLitSelectors; i++) - m_Literals[i].Init(kNumLitSymbols); - const unsigned numItems = (_numDictBits == 0 ? 1 : (_numDictBits << 1)); - const unsigned kNumPosSymbolsMax[kNumMatchSelectors] = { 24, 36, 42 }; - for (i = 0; i < kNumMatchSelectors; i++) - m_PosSlot[i].Init(MyMin(numItems, kNumPosSymbolsMax[i])); - m_LenSlot.Init(kNumLenSymbols); + NumItems = numItems; + ReorderCount = kReorderCount_Start; + UInt16 *freqs = Freqs; + freqs[numItems] = 0; + Byte *vals = Vals; + do + { + *freqs++ = (UInt16)numItems; + *vals++ = (Byte)startVal; + startVal++; + } + while (--numItems); } -HRESULT CDecoder::CodeSpec(const Byte *inData, size_t inSize, UInt32 outSize) +HRESULT CDecoder::Code(const Byte *inData, size_t inSize, UInt32 outSize, bool keepHistory) { if (inSize < 2) return S_FALSE; + if (!keepHistory) + { + _winPos = 0; + m_Selector.Init(kNumSelectors, 0); + unsigned i; + for (i = 0; i < kNumLitSelectors; i++) + m_Literals[i].Init(kNumLitSymbols, i * kNumLitSymbols); + const unsigned numItems = (_numDictBits == 0 ? 1 : (_numDictBits << 1)); + // const unsigned kNumPosSymbolsMax[kNumMatchSelectors] = { 24, 36, 42 }; + for (i = 0; i < kNumMatchSelectors; i++) + { + const unsigned num = 24 + i * 6 + ((i + 1) & 2) * 3; + m_PosSlot[i].Init(MyMin(numItems, num), 0); + } + m_LenSlot.Init(kNumLenSymbols, kMatchMinLen + kNumMatchSelectors - 1); + } CRangeDecoder rc; - rc.Stream.SetStreamAndInit(inData, inSize); - rc.Init(); + rc.Init(inData, inSize); + const UInt32 winSize = _winSize; + Byte *pos; + { + UInt32 winPos = _winPos; + if (winPos == winSize) + { + winPos = 0; + _winPos = winPos; + _overWin = true; + } + if (outSize > winSize - winPos) + return S_FALSE; + pos = _win + winPos; + } while (outSize != 0) { - if (rc.Stream.WasExtraRead()) + if (rc.WasExtraRead()) return S_FALSE; - unsigned selector = m_Selector.Decode(&rc); + const unsigned selector = m_Selector.Decode(&rc); if (selector < kNumLitSelectors) { - const Byte b = (Byte)((selector << (8 - kNumLitSelectorBits)) + m_Literals[selector].Decode(&rc)); - _outWindow.PutByte(b); - outSize--; + const unsigned b = m_Literals[selector].Decode(&rc); + *pos++ = (Byte)b; + --outSize; + // if (--outSize == 0) break; } else { - selector -= kNumLitSelectors; - unsigned len = selector + kMatchMinLen; + unsigned len = selector - kNumLitSelectors + kMatchMinLen; - if (selector == 2) + if (selector == kNumLitSelectors + kNumMatchSelectors - 1) { - unsigned lenSlot = m_LenSlot.Decode(&rc); - if (lenSlot >= kNumSimpleLenSlots) + len = m_LenSlot.Decode(&rc); + if (len >= kNumSimpleLenSlots + kMatchMinLen + kNumMatchSelectors - 1) { - lenSlot -= 2; - const unsigned numDirectBits = (unsigned)(lenSlot >> 2); - len += ((4 | (lenSlot & 3)) << numDirectBits) - 2; + len -= kNumSimpleLenSlots - 4 + kMatchMinLen + kNumMatchSelectors - 1; + const unsigned numDirectBits = (unsigned)(len >> 2); + len = ((4 | (len & 3)) << numDirectBits) - (4 << 1) + + kNumSimpleLenSlots + + kMatchMinLen + kNumMatchSelectors - 1; if (numDirectBits < 6) - len += rc.Stream.ReadBits(numDirectBits); + len += rc.ReadBits(numDirectBits); } - else - len += lenSlot; } - UInt32 dist = m_PosSlot[selector].Decode(&rc); + UInt32 dist = m_PosSlot[(size_t)selector - kNumLitSelectors].Decode(&rc); - if (dist >= kNumSimplePosSlots) + if (dist >= 4) { - unsigned numDirectBits = (unsigned)((dist >> 1) - 1); - dist = ((2 | (dist & 1)) << numDirectBits) + rc.Stream.ReadBits(numDirectBits); + const unsigned numDirectBits = (unsigned)((dist >> 1) - 1); + dist = ((2 | (dist & 1)) << numDirectBits) + rc.ReadBits(numDirectBits); } - unsigned locLen = len; - if (len > outSize) - locLen = (unsigned)outSize; - if (!_outWindow.CopyBlock(dist, locLen)) - return S_FALSE; - outSize -= locLen; - len -= locLen; - if (len != 0) + if ((Int32)(outSize -= len) < 0) return S_FALSE; + + ptrdiff_t srcPos = (ptrdiff_t)(Int32)((pos - _win) - (ptrdiff_t)dist - 1); + if (srcPos < 0) + { + if (!_overWin) + return S_FALSE; + UInt32 rem = (UInt32)-srcPos; + srcPos += winSize; + if (rem < len) + { + const Byte *src = _win + srcPos; + len -= rem; + do + *pos++ = *src++; + while (--rem); + srcPos = 0; + } + } + const Byte *src = _win + srcPos; + do + *pos++ = *src++; + while (--len); + // if (outSize == 0) break; } } + _winPos = (UInt32)(size_t)(pos - _win); return rc.Finish() ? S_OK : S_FALSE; } -HRESULT CDecoder::Code(const Byte *inData, size_t inSize, - ISequentialOutStream *outStream, UInt32 outSize, - bool keepHistory) -{ - try - { - _outWindow.SetStream(outStream); - _outWindow.Init(keepHistory); - if (!keepHistory) - Init(); - - const HRESULT res = CodeSpec(inData, inSize, outSize); - const HRESULT res2 = _outWindow.Flush(); - return res != S_OK ? res : res2; - } - catch(const CLzOutWindowException &e) { return e.ErrorCode; } - catch(...) { return S_FALSE; } -} HRESULT CDecoder::SetParams(unsigned numDictBits) { if (numDictBits > 21) return E_INVALIDARG; _numDictBits = numDictBits; - if (!_outWindow.Create((UInt32)1 << _numDictBits)) - return E_OUTOFMEMORY; + _winPos = 0; + _overWin = false; + + if (numDictBits < 15) + numDictBits = 15; + _winSize = (UInt32)1 << numDictBits; + if (!_win || _winSize > _winSize_allocated) + { + MidFree(_win); + _win = NULL; + _win = (Byte *)MidAlloc(_winSize); + if (!_win) + return E_OUTOFMEMORY; + _winSize_allocated = _winSize; + } return S_OK; } diff --git a/CPP/7zip/Compress/QuantumDecoder.h b/CPP/7zip/Compress/QuantumDecoder.h index 925989a..fd5baea 100644 --- a/CPP/7zip/Compress/QuantumDecoder.h +++ b/CPP/7zip/Compress/QuantumDecoder.h @@ -3,149 +3,41 @@ #ifndef ZIP7_INC_COMPRESS_QUANTUM_DECODER_H #define ZIP7_INC_COMPRESS_QUANTUM_DECODER_H -#include "../../Common/MyCom.h" - -#include "LzOutWindow.h" +#include "../../Common/MyTypes.h" namespace NCompress { namespace NQuantum { -class CBitDecoder -{ - UInt32 Value; - bool _extra; - const Byte *_buf; - const Byte *_bufLim; -public: - void SetStreamAndInit(const Byte *inData, size_t inSize) - { - _buf = inData; - _bufLim = inData + inSize; - Value = 0x10000; - _extra = false; - } - - bool WasExtraRead() const { return _extra; } - - bool WasFinishedOK() const - { - return !_extra && _buf == _bufLim; - } - - UInt32 ReadBit() - { - if (Value >= 0x10000) - { - Byte b; - if (_buf >= _bufLim) - { - b = 0xFF; - _extra = true; - } - else - b = *_buf++; - Value = 0x100 | b; - } - UInt32 res = (Value >> 7) & 1; - Value <<= 1; - return res; - } - - UInt32 ReadStart16Bits() - { - // we use check for extra read in another code. - UInt32 val = ((UInt32)*_buf << 8) | _buf[1]; - _buf += 2; - return val; - } - - UInt32 ReadBits(unsigned numBits) // numBits > 0 - { - UInt32 res = 0; - do - res = (res << 1) | ReadBit(); - while (--numBits); - return res; - } -}; - - -class CRangeDecoder -{ - UInt32 Low; - UInt32 Range; - UInt32 Code; -public: - CBitDecoder Stream; - - void Init() - { - Low = 0; - Range = 0x10000; - Code = Stream.ReadStart16Bits(); - } - - bool Finish() - { - // do all streams use these two bits at end? - if (Stream.ReadBit() != 0) return false; - if (Stream.ReadBit() != 0) return false; - return Stream.WasFinishedOK(); - } - - UInt32 GetThreshold(UInt32 total) const - { - return ((Code + 1) * total - 1) / Range; // & 0xFFFF is not required; - } - - void Decode(UInt32 start, UInt32 end, UInt32 total) - { - UInt32 high = Low + end * Range / total - 1; - UInt32 offset = start * Range / total; - Code -= offset; - Low += offset; - for (;;) - { - if ((Low & 0x8000) != (high & 0x8000)) - { - if ((Low & 0x4000) == 0 || (high & 0x4000) != 0) - break; - Low &= 0x3FFF; - high |= 0x4000; - } - Low = (Low << 1) & 0xFFFF; - high = ((high << 1) | 1) & 0xFFFF; - Code = ((Code << 1) | Stream.ReadBit()); - } - Range = high - Low + 1; - } -}; - - const unsigned kNumLitSelectorBits = 2; -const unsigned kNumLitSelectors = (1 << kNumLitSelectorBits); +const unsigned kNumLitSelectors = 1 << kNumLitSelectorBits; const unsigned kNumLitSymbols = 1 << (8 - kNumLitSelectorBits); const unsigned kNumMatchSelectors = 3; const unsigned kNumSelectors = kNumLitSelectors + kNumMatchSelectors; const unsigned kNumSymbolsMax = kNumLitSymbols; // 64 +class CRangeDecoder; class CModelDecoder { unsigned NumItems; unsigned ReorderCount; - UInt16 Freqs[kNumSymbolsMax + 1]; Byte Vals[kNumSymbolsMax]; + UInt16 Freqs[kNumSymbolsMax + 1]; public: - void Init(unsigned numItems); + Byte _pad[64 - 10]; // for structure size alignment + + void Init(unsigned numItems, unsigned startVal); unsigned Decode(CRangeDecoder *rc); }; -Z7_CLASS_IMP_COM_0( - CDecoder -) - CLzOutWindow _outWindow; +class CDecoder +{ + UInt32 _winSize; + UInt32 _winPos; + UInt32 _winSize_allocated; + bool _overWin; + Byte *_win; unsigned _numDictBits; CModelDecoder m_Selector; @@ -156,12 +48,11 @@ Z7_CLASS_IMP_COM_0( void Init(); HRESULT CodeSpec(const Byte *inData, size_t inSize, UInt32 outSize); public: - HRESULT Code(const Byte *inData, size_t inSize, - ISequentialOutStream *outStream, UInt32 outSize, - bool keepHistory); + HRESULT Code(const Byte *inData, size_t inSize, UInt32 outSize, bool keepHistory); HRESULT SetParams(unsigned numDictBits); - CDecoder(): _numDictBits(0) {} + CDecoder(): _win(NULL), _numDictBits(0) {} + const Byte * GetDataPtr() const { return _win + _winPos; } }; }} diff --git a/CPP/7zip/Compress/Rar1Decoder.h b/CPP/7zip/Compress/Rar1Decoder.h index 6d0e31d..e1e22e2 100644 --- a/CPP/7zip/Compress/Rar1Decoder.h +++ b/CPP/7zip/Compress/Rar1Decoder.h @@ -12,19 +12,22 @@ #include "../Common/InBuffer.h" #include "BitmDecoder.h" -#include "HuffmanDecoder.h" #include "LzOutWindow.h" namespace NCompress { namespace NRar1 { -const UInt32 kNumRepDists = 4; +const unsigned kNumRepDists = 4; Z7_CLASS_IMP_COM_2( CDecoder , ICompressCoder , ICompressSetDecoderProperties2 ) + bool _isSolid; + bool _solidAllowed; + bool StMode; + CLzOutWindow m_OutWindowStream; NBitm::CDecoder m_InBitStream; @@ -36,10 +39,6 @@ Z7_CLASS_IMP_COM_2( UInt32 m_RepDistPtr; UInt32 m_RepDists[kNumRepDists]; - bool _isSolid; - bool _solidAllowed; - - bool StMode; int FlagsCnt; UInt32 FlagBuf, AvrPlc, AvrPlcB, AvrLn1, AvrLn2, AvrLn3; unsigned Buf60, NumHuf, LCount; diff --git a/CPP/7zip/Compress/Rar2Decoder.cpp b/CPP/7zip/Compress/Rar2Decoder.cpp index 92404b6..63c04a1 100644 --- a/CPP/7zip/Compress/Rar2Decoder.cpp +++ b/CPP/7zip/Compress/Rar2Decoder.cpp @@ -93,7 +93,7 @@ CDecoder::CDecoder(): void CDecoder::InitStructures() { m_MmFilter.Init(); - for (unsigned i = 0; i < kNumRepDists; i++) + for (unsigned i = 0; i < kNumReps; i++) m_RepDists[i] = 0; m_RepDistPtr = 0; m_LastLength = 0; @@ -104,10 +104,34 @@ UInt32 CDecoder::ReadBits(unsigned numBits) { return m_InBitStream.ReadBits(numB #define RIF(x) { if (!(x)) return false; } +static const unsigned kRepBothNumber = 256; +static const unsigned kRepNumber = kRepBothNumber + 1; +static const unsigned kLen2Number = kRepNumber + kNumReps; +static const unsigned kReadTableNumber = kLen2Number + kNumLen2Symbols; +static const unsigned kMatchNumber = kReadTableNumber + 1; + +// static const unsigned kDistTableStart = kMainTableSize; +// static const unsigned kLenTableStart = kDistTableStart + kDistTableSize; + +static const UInt32 kDistStart [kDistTableSize] = {0,1,2,3,4,6,8,12,16,24,32,48,64,96,128,192,256,384,512,768,1024,1536,2048,3072,4096,6144,8192,12288,16384,24576,32768U,49152U,65536,98304,131072,196608,262144,327680,393216,458752,524288,589824,655360,720896,786432,851968,917504,983040}; +static const Byte kDistDirectBits[kDistTableSize] = {0,0,0,0,1,1,2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16}; + +static const Byte kLen2DistStarts [kNumLen2Symbols]={0,4,8,16,32,64,128,192}; +static const Byte kLen2DistDirectBits[kNumLen2Symbols]={2,2,3, 4, 5, 6, 6, 6}; + +static const UInt32 kDistLimit2 = 0x101 - 1; +static const UInt32 kDistLimit3 = 0x2000 - 1; +static const UInt32 kDistLimit4 = 0x40000 - 1; + +// static const UInt32 kMatchMaxLen = 255 + 2; +// static const UInt32 kMatchMaxLenMax = 255 + 5; + + bool CDecoder::ReadTables(void) { m_TablesOK = false; + const unsigned kLevelTableSize = 19; Byte levelLevels[kLevelTableSize]; Byte lens[kMaxTableSize]; @@ -123,7 +147,7 @@ bool CDecoder::ReadTables(void) m_NumChannels = ReadBits(2) + 1; if (m_MmFilter.CurrentChannel >= m_NumChannels) m_MmFilter.CurrentChannel = 0; - numLevels = m_NumChannels * kMMTableSize; + numLevels = m_NumChannels * k_MM_TableSize; } else numLevels = kHeapTablesSizesSum; @@ -131,55 +155,48 @@ bool CDecoder::ReadTables(void) unsigned i; for (i = 0; i < kLevelTableSize; i++) levelLevels[i] = (Byte)ReadBits(4); - RIF(m_LevelDecoder.Build(levelLevels)) + NHuffman::CDecoder256 m_LevelDecoder; + RIF(m_LevelDecoder.Build(levelLevels, NHuffman::k_BuildMode_Full)) i = 0; - do { - const UInt32 sym = m_LevelDecoder.Decode(&m_InBitStream); - if (sym < kTableDirectLevels) + const unsigned sym = m_LevelDecoder.DecodeFull(&m_InBitStream); + if (sym < 16) { - lens[i] = (Byte)((sym + m_LastLevels[i]) & kLevelMask); + lens[i] = (Byte)((sym + m_LastLevels[i]) & 15); i++; } +#if 0 + else if (sym >= kLevelTableSize) + return false; +#endif else { - if (sym == kTableLevelRepNumber) + unsigned num; + Byte v; + if (sym == 16) { - unsigned num = ReadBits(2) + 3; if (i == 0) return false; - num += i; - if (num > numLevels) - { - // return false; - num = numLevels; // original unRAR - } - const Byte v = lens[(size_t)i - 1]; - do - lens[i++] = v; - while (i < num); + num = ReadBits(2) + 3; + v = lens[(size_t)i - 1]; } else { - unsigned num; - if (sym == kTableLevel0Number) - num = ReadBits(3) + 3; - else if (sym == kTableLevel0Number2) - num = ReadBits(7) + 11; - else - return false; - num += i; - if (num > numLevels) - { - // return false; - num = numLevels; // original unRAR - } - do - lens[i++] = 0; - while (i < num); + num = (sym - 17) * 4; + num += num + 3 + ReadBits(3 + num); + v = 0; + } + num += i; + if (num > numLevels) + { + // return false; + num = numLevels; // original unRAR } + do + lens[i++] = v; + while (i < num); } } while (i < numLevels); @@ -190,7 +207,7 @@ bool CDecoder::ReadTables(void) if (m_AudioMode) for (i = 0; i < m_NumChannels; i++) { - RIF(m_MMDecoders[i].Build(&lens[i * kMMTableSize])) + RIF(m_MMDecoders[i].Build(&lens[(size_t)i * k_MM_TableSize])) } else { @@ -202,10 +219,10 @@ bool CDecoder::ReadTables(void) memcpy(m_LastLevels, lens, kMaxTableSize); m_TablesOK = true; - return true; } + bool CDecoder::ReadLastTables() { // it differs a little from pure RAR sources; @@ -216,15 +233,15 @@ bool CDecoder::ReadLastTables() { if (m_AudioMode) { - const UInt32 symbol = m_MMDecoders[m_MmFilter.CurrentChannel].Decode(&m_InBitStream); + const unsigned symbol = m_MMDecoders[m_MmFilter.CurrentChannel].Decode(&m_InBitStream); if (symbol == 256) return ReadTables(); - if (symbol >= kMMTableSize) + if (symbol >= k_MM_TableSize) return false; } else { - const UInt32 sym = m_MainDecoder.Decode(&m_InBitStream); + const unsigned sym = m_MainDecoder.Decode(&m_InBitStream); if (sym == kReadTableNumber) return ReadTables(); if (sym >= kMainTableSize) @@ -239,7 +256,7 @@ bool CDecoder::DecodeMm(UInt32 pos) { while (pos-- != 0) { - const UInt32 symbol = m_MMDecoders[m_MmFilter.CurrentChannel].Decode(&m_InBitStream); + const unsigned symbol = m_MMDecoders[m_MmFilter.CurrentChannel].Decode(&m_InBitStream); if (m_InBitStream.ExtraBitsWereRead()) return false; if (symbol >= 256) @@ -257,14 +274,23 @@ bool CDecoder::DecodeMm(UInt32 pos) return true; } + +typedef unsigned CLenType; + +static inline CLenType SlotToLen(CBitDecoder &_bitStream, CLenType slot) +{ + const unsigned numBits = ((unsigned)slot >> 2) - 1; + return ((4 | (slot & 3)) << numBits) + (CLenType)_bitStream.ReadBits(numBits); +} + bool CDecoder::DecodeLz(Int32 pos) { while (pos > 0) { - UInt32 sym = m_MainDecoder.Decode(&m_InBitStream); + unsigned sym = m_MainDecoder.Decode(&m_InBitStream); if (m_InBitStream.ExtraBitsWereRead()) return false; - UInt32 length, distance; + UInt32 len, distance; if (sym < 256) { m_OutWindowStream.PutByte(Byte(sym)); @@ -275,44 +301,49 @@ bool CDecoder::DecodeLz(Int32 pos) { if (sym >= kMainTableSize) return false; - sym -= kMatchNumber; - length = kNormalMatchMinLen + UInt32(kLenStart[sym]) + - m_InBitStream.ReadBits(kLenDirectBits[sym]); + len = sym - kMatchNumber; + if (len >= 8) + len = SlotToLen(m_InBitStream, len); + len += 3; + sym = m_DistDecoder.Decode(&m_InBitStream); if (sym >= kDistTableSize) return false; distance = kDistStart[sym] + m_InBitStream.ReadBits(kDistDirectBits[sym]); if (distance >= kDistLimit3) { - length += 2 - ((distance - kDistLimit4) >> 31); - // length++; + len += 2 - ((distance - kDistLimit4) >> 31); + // len++; // if (distance >= kDistLimit4) - // length++; + // len++; } } else if (sym == kRepBothNumber) { - length = m_LastLength; - if (length == 0) + len = m_LastLength; + if (len == 0) return false; distance = m_RepDists[(m_RepDistPtr + 4 - 1) & 3]; } else if (sym < kLen2Number) { distance = m_RepDists[(m_RepDistPtr - (sym - kRepNumber + 1)) & 3]; - sym = m_LenDecoder.Decode(&m_InBitStream); - if (sym >= kLenTableSize) + len = m_LenDecoder.Decode(&m_InBitStream); + if (len >= kLenTableSize) return false; - length = 2 + kLenStart[sym] + m_InBitStream.ReadBits(kLenDirectBits[sym]); + if (len >= 8) + len = SlotToLen(m_InBitStream, len); + len += 2; + if (distance >= kDistLimit2) { - length++; + len++; if (distance >= kDistLimit3) { - length += 2 - ((distance - kDistLimit4) >> 31); - // length++; + len += 2 - ((distance - kDistLimit4) >> 31); + // len++; // if (distance >= kDistLimit4) - // length++; + // len++; } } } @@ -321,16 +352,16 @@ bool CDecoder::DecodeLz(Int32 pos) sym -= kLen2Number; distance = kLen2DistStarts[sym] + m_InBitStream.ReadBits(kLen2DistDirectBits[sym]); - length = 2; + len = 2; } else // (sym == kReadTableNumber) return true; m_RepDists[m_RepDistPtr++ & 3] = distance; - m_LastLength = length; - if (!m_OutWindowStream.CopyBlock(distance, length)) + m_LastLength = len; + if (!m_OutWindowStream.CopyBlock(distance, len)) return false; - pos -= length; + pos -= len; } return true; } diff --git a/CPP/7zip/Compress/Rar2Decoder.h b/CPP/7zip/Compress/Rar2Decoder.h index d77bfc1..c9ddedb 100644 --- a/CPP/7zip/Compress/Rar2Decoder.h +++ b/CPP/7zip/Compress/Rar2Decoder.h @@ -18,59 +18,16 @@ namespace NCompress { namespace NRar2 { -const unsigned kNumRepDists = 4; +const unsigned kNumReps = 4; const unsigned kDistTableSize = 48; - -const unsigned kMMTableSize = 256 + 1; - -const UInt32 kMainTableSize = 298; -const UInt32 kLenTableSize = 28; - -const UInt32 kDistTableStart = kMainTableSize; -const UInt32 kLenTableStart = kDistTableStart + kDistTableSize; - -const UInt32 kHeapTablesSizesSum = kMainTableSize + kDistTableSize + kLenTableSize; - -const UInt32 kLevelTableSize = 19; - -const UInt32 kMMTablesSizesSum = kMMTableSize * 4; - -const UInt32 kMaxTableSize = kMMTablesSizesSum; - -const UInt32 kTableDirectLevels = 16; -const UInt32 kTableLevelRepNumber = kTableDirectLevels; -const UInt32 kTableLevel0Number = kTableLevelRepNumber + 1; -const UInt32 kTableLevel0Number2 = kTableLevel0Number + 1; - -const UInt32 kLevelMask = 0xF; - - -const UInt32 kRepBothNumber = 256; -const UInt32 kRepNumber = kRepBothNumber + 1; -const UInt32 kLen2Number = kRepNumber + 4; - -const UInt32 kLen2NumNumbers = 8; -const UInt32 kReadTableNumber = kLen2Number + kLen2NumNumbers; -const UInt32 kMatchNumber = kReadTableNumber + 1; - -const Byte kLenStart [kLenTableSize] = {0,1,2,3,4,5,6,7,8,10,12,14,16,20,24,28,32,40,48,56,64,80,96,112,128,160,192,224}; -const Byte kLenDirectBits[kLenTableSize] = {0,0,0,0,0,0,0,0,1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5}; - -const UInt32 kDistStart [kDistTableSize] = {0,1,2,3,4,6,8,12,16,24,32,48,64,96,128,192,256,384,512,768,1024,1536,2048,3072,4096,6144,8192,12288,16384,24576,32768U,49152U,65536,98304,131072,196608,262144,327680,393216,458752,524288,589824,655360,720896,786432,851968,917504,983040}; -const Byte kDistDirectBits[kDistTableSize] = {0,0,0,0,1,1,2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16}; - -const Byte kLevelDirectBits[kLevelTableSize] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 7}; - -const Byte kLen2DistStarts[kLen2NumNumbers]={0,4,8,16,32,64,128,192}; -const Byte kLen2DistDirectBits[kLen2NumNumbers]={2,2,3, 4, 5, 6, 6, 6}; - -const UInt32 kDistLimit2 = 0x101 - 1; -const UInt32 kDistLimit3 = 0x2000 - 1; -const UInt32 kDistLimit4 = 0x40000 - 1; - -const UInt32 kMatchMaxLen = 255 + 2; -const UInt32 kMatchMaxLenMax = 255 + 5; -const UInt32 kNormalMatchMinLen = 3; +const unsigned kNumLen2Symbols = 8; +const unsigned kLenTableSize = 28; +const unsigned kMainTableSize = 256 + 2 + kNumReps + kNumLen2Symbols + kLenTableSize; +const unsigned kHeapTablesSizesSum = kMainTableSize + kDistTableSize + kLenTableSize; +const unsigned k_MM_TableSize = 256 + 1; +const unsigned k_MM_NumChanelsMax = 4; +const unsigned k_MM_TablesSizesSum = k_MM_TableSize * k_MM_NumChanelsMax; +const unsigned kMaxTableSize = k_MM_TablesSizesSum; namespace NMultimedia { @@ -83,18 +40,13 @@ struct CFilter UInt32 ByteCount; int LastChar; - Byte Decode(int &channelDelta, Byte delta); - void Init() { memset(this, 0, sizeof(*this)); } - + Byte Decode(int &channelDelta, Byte delta); }; -const unsigned kNumChanelsMax = 4; - -class CFilter2 +struct CFilter2 { -public: - CFilter m_Filters[kNumChanelsMax]; + CFilter m_Filters[k_MM_NumChanelsMax]; int m_ChannelDelta; unsigned CurrentChannel; @@ -103,47 +55,42 @@ class CFilter2 { return m_Filters[CurrentChannel].Decode(m_ChannelDelta, delta); } - }; } typedef NBitm::CDecoder CBitDecoder; -const unsigned kNumHuffmanBits = 15; +const unsigned kNumHufBits = 15; Z7_CLASS_IMP_NOQIB_2( CDecoder , ICompressCoder , ICompressSetDecoderProperties2 ) + bool _isSolid; + bool _solidAllowed; + bool m_TablesOK; + bool m_AudioMode; + CLzOutWindow m_OutWindowStream; CBitDecoder m_InBitStream; UInt32 m_RepDistPtr; - UInt32 m_RepDists[kNumRepDists]; - + UInt32 m_RepDists[kNumReps]; UInt32 m_LastLength; + unsigned m_NumChannels; - bool _isSolid; - bool _solidAllowed; - bool m_TablesOK; - bool m_AudioMode; - - NHuffman::CDecoder m_MainDecoder; - NHuffman::CDecoder m_DistDecoder; - NHuffman::CDecoder m_LenDecoder; - NHuffman::CDecoder m_MMDecoders[NMultimedia::kNumChanelsMax]; - NHuffman::CDecoder m_LevelDecoder; + NHuffman::CDecoder m_MainDecoder; + NHuffman::CDecoder256 m_DistDecoder; + NHuffman::CDecoder256 m_LenDecoder; + NHuffman::CDecoder m_MMDecoders[k_MM_NumChanelsMax]; UInt64 m_PackSize; - - unsigned m_NumChannels; + NMultimedia::CFilter2 m_MmFilter; - Byte m_LastLevels[kMaxTableSize]; - void InitStructures(); UInt32 ReadBits(unsigned numBits); bool ReadTables(); diff --git a/CPP/7zip/Compress/Rar3Decoder.cpp b/CPP/7zip/Compress/Rar3Decoder.cpp index a4943ad..c17d46d 100644 --- a/CPP/7zip/Compress/Rar3Decoder.cpp +++ b/CPP/7zip/Compress/Rar3Decoder.cpp @@ -17,8 +17,8 @@ namespace NRar3 { static const UInt32 kNumAlignReps = 15; -static const UInt32 kSymbolReadTable = 256; -static const UInt32 kSymbolRep = 259; +static const unsigned kSymbolReadTable = 256; +static const unsigned kSymbolRep = 259; static const Byte kDistDirectBits[kDistTableSize] = {0,0,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15, @@ -54,15 +54,15 @@ static Byte Wrap_ReadBits8(IByteInPtr pp) throw() CDecoder::CDecoder(): + _isSolid(false), + _solidAllowed(false), _window(NULL), _winPos(0), _wrPtr(0), _lzSize(0), _writtenFileSize(0), _vmData(NULL), - _vmCode(NULL), - _isSolid(false), - _solidAllowed(false) + _vmCode(NULL) { Ppmd7_Construct(&_ppmd); @@ -252,8 +252,8 @@ bool CDecoder::AddVmCode(UInt32 firstByte, UInt32 codeSize) if (_numEmptyTempFilters != 0) { - unsigned num = _tempFilters.Size(); - CTempFilter **tempFilters = &_tempFilters.Front(); + const unsigned num = _tempFilters.Size(); + CTempFilter **tempFilters = _tempFilters.NonConstData(); unsigned w = 0; for (unsigned i = 0; i < num; i++) @@ -542,6 +542,7 @@ HRESULT CDecoder::ReadTables(bool &keepDecompressing) PrevAlignBits = 0; PrevAlignCount = 0; + const unsigned kLevelTableSize = 20; Byte levelLevels[kLevelTableSize]; Byte lens[kTablesSizesSum]; @@ -568,23 +569,26 @@ HRESULT CDecoder::ReadTables(bool &keepDecompressing) levelLevels[i] = (Byte)len; } - RIF(m_LevelDecoder.Build(levelLevels)) + NHuffman::CDecoder256 m_LevelDecoder; + RIF(m_LevelDecoder.Build(levelLevels, NHuffman::k_BuildMode_Full)) i = 0; do { - const UInt32 sym = m_LevelDecoder.Decode(&m_InBitStream.BitDecoder); + const unsigned sym = m_LevelDecoder.DecodeFull(&m_InBitStream.BitDecoder); if (sym < 16) { lens[i] = Byte((sym + m_LastLevels[i]) & 15); i++; } +#if 0 else if (sym > kLevelTableSize) return S_FALSE; +#endif else { - unsigned num = ((sym - 16) & 1) * 4; + unsigned num = ((sym /* - 16 */) & 1) * 4; num += num + 3 + (unsigned)ReadBits(num + 3); num += i; if (num > kTablesSizesSum) @@ -678,7 +682,7 @@ HRESULT CDecoder::DecodeLZ(bool &keepDecompressing) if (InputEofError_Fast()) return S_FALSE; - UInt32 sym = m_MainDecoder.Decode(&m_InBitStream.BitDecoder); + unsigned sym = m_MainDecoder.Decode(&m_InBitStream.BitDecoder); if (sym < 256) { PutByte((Byte)sym); @@ -722,14 +726,14 @@ HRESULT CDecoder::DecodeLZ(bool &keepDecompressing) rep0 = dist; } - const UInt32 sym2 = m_LenDecoder.Decode(&m_InBitStream.BitDecoder); + const unsigned sym2 = m_LenDecoder.Decode(&m_InBitStream.BitDecoder); if (sym2 >= kLenTableSize) return S_FALSE; len = 2 + sym2; if (sym2 >= 8) { - unsigned num = (sym2 >> 2) - 1; - len = 2 + ((4 + (sym2 & 3)) << num) + m_InBitStream.BitDecoder.ReadBits_upto8(num); + const unsigned num = (sym2 >> 2) - 1; + len = 2 + (UInt32)((4 + (sym2 & 3)) << num) + m_InBitStream.BitDecoder.ReadBits_upto8(num); } } else @@ -750,9 +754,9 @@ HRESULT CDecoder::DecodeLZ(bool &keepDecompressing) if (sym >= 8) { const unsigned num = (sym >> 2) - 1; - len = kNormalMatchMinLen + ((4 + (sym & 3)) << num) + m_InBitStream.BitDecoder.ReadBits_upto8(num); + len = kNormalMatchMinLen + (UInt32)((4 + (sym & 3)) << num) + m_InBitStream.BitDecoder.ReadBits_upto8(num); } - const UInt32 sym2 = m_DistDecoder.Decode(&m_InBitStream.BitDecoder); + const unsigned sym2 = m_DistDecoder.Decode(&m_InBitStream.BitDecoder); if (sym2 >= kDistTableSize) return S_FALSE; rep0 = kDistStart[sym2]; @@ -768,7 +772,7 @@ HRESULT CDecoder::DecodeLZ(bool &keepDecompressing) } else { - const UInt32 sym3 = m_AlignDecoder.Decode(&m_InBitStream.BitDecoder); + const unsigned sym3 = m_AlignDecoder.Decode(&m_InBitStream.BitDecoder); if (sym3 < (1 << kNumAlignBits)) { rep0 += sym3; diff --git a/CPP/7zip/Compress/Rar3Decoder.h b/CPP/7zip/Compress/Rar3Decoder.h index 4711052..48fc212 100644 --- a/CPP/7zip/Compress/Rar3Decoder.h +++ b/CPP/7zip/Compress/Rar3Decoder.h @@ -22,21 +22,21 @@ namespace NCompress { namespace NRar3 { +const unsigned kNumHuffmanBits = 15; + const UInt32 kWindowSize = 1 << 22; -const UInt32 kWindowMask = (kWindowSize - 1); +const UInt32 kWindowMask = kWindowSize - 1; -const UInt32 kNumReps = 4; -const UInt32 kNumLen2Symbols = 8; -const UInt32 kLenTableSize = 28; -const UInt32 kMainTableSize = 256 + 1 + 1 + 1 + kNumReps + kNumLen2Symbols + kLenTableSize; -const UInt32 kDistTableSize = 60; +const unsigned kNumReps = 4; +const unsigned kNumLen2Symbols = 8; +const unsigned kLenTableSize = 28; +const unsigned kMainTableSize = 256 + 3 + kNumReps + kNumLen2Symbols + kLenTableSize; +const unsigned kDistTableSize = 60; const unsigned kNumAlignBits = 4; -const UInt32 kAlignTableSize = (1 << kNumAlignBits) + 1; +const unsigned kAlignTableSize = (1 << kNumAlignBits) + 1; -const UInt32 kLevelTableSize = 20; - -const UInt32 kTablesSizesSum = kMainTableSize + kDistTableSize + kAlignTableSize + kLenTableSize; +const unsigned kTablesSizesSum = kMainTableSize + kDistTableSize + kAlignTableSize + kLenTableSize; class CBitDecoder { @@ -68,6 +68,7 @@ class CBitDecoder _value = _value & ((1 << _bitPos) - 1); } + Z7_FORCE_INLINE UInt32 GetValue(unsigned numBits) { if (_bitPos < numBits) @@ -82,7 +83,15 @@ class CBitDecoder } return _value >> (_bitPos - numBits); } + + Z7_FORCE_INLINE + UInt32 GetValue_InHigh32bits() + { + return GetValue(kNumHuffmanBits) << (32 - kNumHuffmanBits); + } + + Z7_FORCE_INLINE void MovePos(unsigned numBits) { _bitPos -= numBits; @@ -91,7 +100,7 @@ class CBitDecoder UInt32 ReadBits(unsigned numBits) { - UInt32 res = GetValue(numBits); + const UInt32 res = GetValue(numBits); MovePos(numBits); return res; } @@ -104,7 +113,7 @@ class CBitDecoder _value = (_value << 8) | Stream.ReadByte(); } _bitPos -= numBits; - UInt32 res = _value >> _bitPos; + const UInt32 res = _value >> _bitPos; _value = _value & ((1 << _bitPos) - 1); return res; } @@ -113,8 +122,8 @@ class CBitDecoder { if (_bitPos == 0) return Stream.ReadByte(); - unsigned bitsPos = _bitPos - 8; - Byte b = (Byte)(_value >> bitsPos); + const unsigned bitsPos = _bitPos - 8; + const Byte b = (Byte)(_value >> bitsPos); _value = _value & ((1 << bitsPos) - 1); _bitPos = bitsPos; return b; @@ -154,13 +163,19 @@ struct CTempFilter: public NVm::CProgramInitState } }; -const unsigned kNumHuffmanBits = 15; Z7_CLASS_IMP_NOQIB_2( CDecoder , ICompressCoder , ICompressSetDecoderProperties2 ) + bool _isSolid; + bool _solidAllowed; + // bool _errorMode; + + bool _lzMode; + bool _unsupportedFilter; + CByteIn m_InBitStream; Byte *_window; UInt32 _winPos; @@ -169,12 +184,12 @@ Z7_CLASS_IMP_NOQIB_2( UInt64 _unpackSize; UInt64 _writtenFileSize; // if it's > _unpackSize, then _unpackSize only written ISequentialOutStream *_outStream; - NHuffman::CDecoder m_MainDecoder; + + NHuffman::CDecoder m_MainDecoder; UInt32 kDistStart[kDistTableSize]; - NHuffman::CDecoder m_DistDecoder; - NHuffman::CDecoder m_AlignDecoder; - NHuffman::CDecoder m_LenDecoder; - NHuffman::CDecoder m_LevelDecoder; + NHuffman::CDecoder256 m_DistDecoder; + NHuffman::CDecoder256 m_AlignDecoder; + NHuffman::CDecoder256 m_LenDecoder; UInt32 _reps[kNumReps]; UInt32 _lastLength; @@ -189,22 +204,15 @@ Z7_CLASS_IMP_NOQIB_2( unsigned _numEmptyTempFilters; UInt32 _lastFilter; - bool _isSolid; - bool _solidAllowed; - // bool _errorMode; - - bool _lzMode; - bool _unsupportedFilter; - UInt32 PrevAlignBits; UInt32 PrevAlignCount; bool TablesRead; bool TablesOK; + bool PpmError; - CPpmd7 _ppmd; int PpmEscChar; - bool PpmError; + CPpmd7 _ppmd; HRESULT WriteDataToStream(const Byte *data, UInt32 size); HRESULT WriteData(const Byte *data, UInt32 size); diff --git a/CPP/7zip/Compress/Rar3Vm.cpp b/CPP/7zip/Compress/Rar3Vm.cpp index 9b728ef..f4ca500 100644 --- a/CPP/7zip/Compress/Rar3Vm.cpp +++ b/CPP/7zip/Compress/Rar3Vm.cpp @@ -967,13 +967,13 @@ static void RgbDecode(Byte *srcData, UInt32 dataSize, UInt32 width, UInt32 posR) for (UInt32 i = curChannel; i < dataSize; i += kNumChannels) { - unsigned int predicted; + unsigned predicted; if (i < width) predicted = prevByte; else { - const unsigned int upperLeftByte = destData[i - width]; - const unsigned int upperByte = destData[i - width + 3]; + const unsigned upperLeftByte = destData[i - width]; + const unsigned upperByte = destData[i - width + 3]; predicted = prevByte + upperByte - upperLeftByte; const int pa = abs((int)(predicted - prevByte)); const int pb = abs((int)(predicted - upperByte)); diff --git a/CPP/7zip/Compress/Rar5Decoder.cpp b/CPP/7zip/Compress/Rar5Decoder.cpp index e55d7de..8be24e2 100644 --- a/CPP/7zip/Compress/Rar5Decoder.cpp +++ b/CPP/7zip/Compress/Rar5Decoder.cpp @@ -4,20 +4,618 @@ #include "StdAfx.h" +#define DICT_SIZE_MAX ((UInt64)1 << DICT_SIZE_BITS_MAX) + +// #include // SSE2 +// #endif + +#include "../../../C/CpuArch.h" +#if 0 +#include "../../../C/Bra.h" +#endif + +#if defined(MY_CPU_ARM64) +#include +#endif + +// #define Z7_RAR5_SHOW_STAT // #include +#ifdef Z7_RAR5_SHOW_STAT +#include +#endif #include "../Common/StreamUtils.h" #include "Rar5Decoder.h" +/* +Note: original-unrar claims that encoder has limitation for Distance: + (Distance <= MaxWinSize - MAX_INC_LZ_MATCH) + MAX_INC_LZ_MATCH = 0x1001 + 3; +*/ + +#define LZ_ERROR_TYPE_NO 0 +#define LZ_ERROR_TYPE_HEADER 1 +// #define LZ_ERROR_TYPE_SYM 1 +#define LZ_ERROR_TYPE_DIST 2 + +static +void My_ZeroMemory(void *p, size_t size) +{ + #if defined(MY_CPU_AMD64) && !defined(_M_ARM64EC) \ + && defined(Z7_MSC_VER_ORIGINAL) && (Z7_MSC_VER_ORIGINAL <= 1400) + // __stosq((UInt64 *)(void *)win, 0, size / 8); + /* + printf("\n__stosb \n"); + #define STEP_BIG (1 << 28) + for (size_t i = 0; i < ((UInt64)1 << 50); i += STEP_BIG) + { + printf("\n__stosb end %p\n", (void *)i); + __stosb((Byte *)p + i, 0, STEP_BIG); + } + */ + // __stosb((Byte *)p, 0, 0); + __stosb((Byte *)p, 0, size); + #else + // SecureZeroMemory (win, STEP); + // ZeroMemory(win, STEP); + // memset(win, 0, STEP); + memset(p, 0, size); + #endif +} + + + +#ifdef MY_CPU_LE_UNALIGN + #define Z7_RAR5_DEC_USE_UNALIGNED_COPY +#endif + +#ifdef Z7_RAR5_DEC_USE_UNALIGNED_COPY + + #define COPY_CHUNK_SIZE 16 + + #define COPY_CHUNK_4_2(dest, src) \ + { \ + ((UInt32 *)(void *)dest)[0] = ((const UInt32 *)(const void *)src)[0]; \ + ((UInt32 *)(void *)dest)[1] = ((const UInt32 *)(const void *)src)[1]; \ + src += 4 * 2; \ + dest += 4 * 2; \ + } + + /* sse2 doesn't help here in GCC and CLANG. + so we disabled sse2 here */ +#if 0 + #if defined(MY_CPU_AMD64) + #define Z7_RAR5_DEC_USE_SSE2 + #elif defined(MY_CPU_X86) + #if defined(_MSC_VER) && _MSC_VER >= 1300 && defined(_M_IX86_FP) && (_M_IX86_FP >= 2) \ + || defined(__SSE2__) \ + // || 1 == 1 // for debug only + #define Z7_RAR5_DEC_USE_SSE2 + #endif + #endif +#endif + + #if defined(MY_CPU_ARM64) + + #define COPY_OFFSET_MIN 16 + #define COPY_CHUNK1(dest, src) \ + { \ + vst1q_u8((uint8_t *)(void *)dest, \ + vld1q_u8((const uint8_t *)(const void *)src)); \ + src += 16; \ + dest += 16; \ + } + + #define COPY_CHUNK(dest, src) \ + { \ + COPY_CHUNK1(dest, src) \ + if (dest >= lim) break; \ + COPY_CHUNK1(dest, src) \ + } + + #elif defined(Z7_RAR5_DEC_USE_SSE2) + #include // sse2 + #define COPY_OFFSET_MIN 16 + + #define COPY_CHUNK1(dest, src) \ + { \ + _mm_storeu_si128((__m128i *)(void *)dest, \ + _mm_loadu_si128((const __m128i *)(const void *)src)); \ + src += 16; \ + dest += 16; \ + } + + #define COPY_CHUNK(dest, src) \ + { \ + COPY_CHUNK1(dest, src) \ + if (dest >= lim) break; \ + COPY_CHUNK1(dest, src) \ + } + + #elif defined(MY_CPU_64BIT) + #define COPY_OFFSET_MIN 8 + + #define COPY_CHUNK(dest, src) \ + { \ + ((UInt64 *)(void *)dest)[0] = ((const UInt64 *)(const void *)src)[0]; \ + src += 8 * 1; dest += 8 * 1; \ + ((UInt64 *)(void *)dest)[0] = ((const UInt64 *)(const void *)src)[0]; \ + src += 8 * 1; dest += 8 * 1; \ + } + + #else + #define COPY_OFFSET_MIN 4 + + #define COPY_CHUNK(dest, src) \ + { \ + COPY_CHUNK_4_2(dest, src); \ + COPY_CHUNK_4_2(dest, src); \ + } + + #endif +#endif + + +#ifndef COPY_CHUNK_SIZE + #define COPY_OFFSET_MIN 4 + #define COPY_CHUNK_SIZE 8 + #define COPY_CHUNK_2(dest, src) \ + { \ + const Byte a0 = src[0]; \ + const Byte a1 = src[1]; \ + dest[0] = a0; \ + dest[1] = a1; \ + src += 2; \ + dest += 2; \ + } + #define COPY_CHUNK(dest, src) \ + { \ + COPY_CHUNK_2(dest, src) \ + COPY_CHUNK_2(dest, src) \ + COPY_CHUNK_2(dest, src) \ + COPY_CHUNK_2(dest, src) \ + } +#endif + + +#define COPY_CHUNKS \ +{ \ + Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE \ + do { COPY_CHUNK(dest, src) } \ + while (dest < lim); \ +} + namespace NCompress { namespace NRar5 { +typedef +#if 1 + unsigned +#else + size_t +#endif + CLenType; + +// (len != 0) +static +Z7_FORCE_INLINE +// Z7_ATTRIB_NO_VECTOR +void CopyMatch(size_t offset, Byte *dest, const Byte *src, const Byte *lim) +{ + { + // (COPY_OFFSET_MIN >= 4) + if (offset >= COPY_OFFSET_MIN) + { + COPY_CHUNKS + // return; + } + else + #if (COPY_OFFSET_MIN > 4) + #if COPY_CHUNK_SIZE < 8 + #error Stop_Compiling_Bad_COPY_CHUNK_SIZE + #endif + if (offset >= 4) + { + Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE + do + { + COPY_CHUNK_4_2(dest, src) + #if COPY_CHUNK_SIZE < 16 + if (dest >= lim) break; + #endif + COPY_CHUNK_4_2(dest, src) + } + while (dest < lim); + // return; + } + else + #endif + { + // (offset < 4) + const unsigned b0 = src[0]; + if (offset < 2) + { + #if defined(Z7_RAR5_DEC_USE_UNALIGNED_COPY) && (COPY_CHUNK_SIZE == 16) + #if defined(MY_CPU_64BIT) + { + const UInt64 v64 = (UInt64)b0 * 0x0101010101010101; + Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE + do + { + ((UInt64 *)(void *)dest)[0] = v64; + ((UInt64 *)(void *)dest)[1] = v64; + dest += 16; + } + while (dest < lim); + } + #else + { + UInt32 v = b0; + v |= v << 8; + v |= v << 16; + do + { + ((UInt32 *)(void *)dest)[0] = v; + ((UInt32 *)(void *)dest)[1] = v; + dest += 8; + ((UInt32 *)(void *)dest)[0] = v; + ((UInt32 *)(void *)dest)[1] = v; + dest += 8; + } + while (dest < lim); + } + #endif + #else + do + { + dest[0] = (Byte)b0; + dest[1] = (Byte)b0; + dest += 2; + dest[0] = (Byte)b0; + dest[1] = (Byte)b0; + dest += 2; + } + while (dest < lim); + #endif + } + else if (offset == 2) + { + const Byte b1 = src[1]; + { + do + { + dest[0] = (Byte)b0; + dest[1] = b1; + dest += 2; + } + while (dest < lim); + } + } + else // (offset == 3) + { + const Byte b1 = src[1]; + const Byte b2 = src[2]; + do + { + dest[0] = (Byte)b0; + dest[1] = b1; + dest[2] = b2; + dest += 3; + } + while (dest < lim); + } + } + } +} + static const size_t kInputBufSize = 1 << 20; - +static const UInt32 k_Filter_BlockSize_MAX = 1 << 22; +static const unsigned k_Filter_AfterPad_Size = 64; + +#ifdef Z7_RAR5_SHOW_STAT +static const unsigned kNumStats1 = 10; +static const unsigned kNumStats2 = (1 << 12) + 16; +static UInt32 g_stats1[kNumStats1]; +static UInt32 g_stats2[kNumStats1][kNumStats2]; +#endif + +#if 1 +MY_ALIGN(32) +// DICT_SIZE_BITS_MAX-1 are required +static const Byte k_LenPlusTable[DICT_SIZE_BITS_MAX] = + { 0,0,0,0,0,0,0,1,1,1,1,1,2,2,2,2,2,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3 }; +#endif + + + +class CBitDecoder +{ +public: + const Byte *_buf; + const Byte *_bufCheck_Block; // min(ptr for _blockEnd, _bufCheck) + unsigned _bitPos; // = [0 ... 7] + bool _wasFinished; + bool _minorError; + unsigned _blockEndBits7; // = [0 ... 7] : the number of additional bits in (_blockEnd) poisition. + HRESULT _hres; + const Byte *_bufCheck; // relaxed limit (16 bytes before real end of input data in buffer) + Byte *_bufLim; // end if input data + Byte *_bufBase; + ISequentialInStream *_stream; + + UInt64 _processedSize; + UInt64 _blockEnd; // absolute end of current block + // but it doesn't include additional _blockEndBits7 [0 ... 7] bits + + Z7_FORCE_INLINE + void CopyFrom(const CBitDecoder &a) + { + _buf = a._buf; + _bufCheck_Block = a._bufCheck_Block; + _bitPos = a._bitPos; + _wasFinished = a._wasFinished; + _blockEndBits7 = a._blockEndBits7; + _bufCheck = a._bufCheck; + _bufLim = a._bufLim; + _bufBase = a._bufBase; + + _processedSize = a._processedSize; + _blockEnd = a._blockEnd; + } + + Z7_FORCE_INLINE + void RestoreFrom2(const CBitDecoder &a) + { + _buf = a._buf; + _bitPos = a._bitPos; + } + + Z7_FORCE_INLINE + void SetCheck_forBlock() + { + _bufCheck_Block = _bufCheck; + if (_bufCheck > _buf) + { + const UInt64 processed = GetProcessedSize_Round(); + if (_blockEnd < processed) + _bufCheck_Block = _buf; + else + { + const UInt64 delta = _blockEnd - processed; + if ((size_t)(_bufCheck - _buf) > delta) + _bufCheck_Block = _buf + (size_t)delta; + } + } + } + + Z7_FORCE_INLINE + bool IsBlockOverRead() const + { + const UInt64 v = GetProcessedSize_Round(); + if (v < _blockEnd) return false; + if (v > _blockEnd) return true; + return _bitPos > _blockEndBits7; + } + + /* + CBitDecoder() throw(): + _buf(0), + _bufLim(0), + _bufBase(0), + _stream(0), + _processedSize(0), + _wasFinished(false) + {} + */ + + Z7_FORCE_INLINE + void Init() throw() + { + _blockEnd = 0; + _blockEndBits7 = 0; + + _bitPos = 0; + _processedSize = 0; + _buf = _bufBase; + _bufLim = _bufBase; + _bufCheck = _buf; + _bufCheck_Block = _buf; + _wasFinished = false; + _minorError = false; + } + + void Prepare2() throw(); + + Z7_FORCE_INLINE + void Prepare() throw() + { + if (_buf >= _bufCheck) + Prepare2(); + } + + Z7_FORCE_INLINE + bool ExtraBitsWereRead() const + { + return _buf >= _bufLim && (_buf > _bufLim || _bitPos != 0); + } + + Z7_FORCE_INLINE bool InputEofError() const { return ExtraBitsWereRead(); } + + Z7_FORCE_INLINE unsigned GetProcessedBits7() const { return _bitPos; } + Z7_FORCE_INLINE UInt64 GetProcessedSize_Round() const { return _processedSize + (size_t)(_buf - _bufBase); } + Z7_FORCE_INLINE UInt64 GetProcessedSize() const { return _processedSize + (size_t)(_buf - _bufBase) + ((_bitPos + 7) >> 3); } + + Z7_FORCE_INLINE + void AlignToByte() + { + if (_bitPos != 0) + { +#if 1 + // optional check of unused bits for strict checking: + // original-unrar doesn't check it: + const unsigned b = (unsigned)*_buf << _bitPos; + if (b & 0xff) + _minorError = true; +#endif + _buf++; + _bitPos = 0; + } + // _buf += (_bitPos + 7) >> 3; + // _bitPos = 0; + } + + Z7_FORCE_INLINE + Byte ReadByte_InAligned() + { + return *_buf++; + } + + Z7_FORCE_INLINE + UInt32 GetValue(unsigned numBits) const + { + // 0 < numBits <= 17 : supported values +#if defined(Z7_CPU_FAST_BSWAP_SUPPORTED) && defined(MY_CPU_LE_UNALIGN) + UInt32 v = GetBe32(_buf); +#if 1 + return (v >> (32 - numBits - _bitPos)) & ((1u << numBits) - 1); +#else + return (v << _bitPos) >> (32 - numBits); +#endif +#else + UInt32 v = ((UInt32)_buf[0] << 16) | ((UInt32)_buf[1] << 8) | (UInt32)_buf[2]; + v >>= 24 - numBits - _bitPos; + return v & ((1 << numBits) - 1); +#endif + } + + Z7_FORCE_INLINE + UInt32 GetValue_InHigh32bits() const + { + // 0 < numBits <= 17 : supported vales +#if defined(Z7_CPU_FAST_BSWAP_SUPPORTED) && defined(MY_CPU_LE_UNALIGN) + return GetBe32(_buf) << _bitPos; +#else + const UInt32 v = ((UInt32)_buf[0] << 16) | ((UInt32)_buf[1] << 8) | (UInt32)_buf[2]; + return v << (_bitPos + 8); +#endif + } + + + Z7_FORCE_INLINE + void MovePos(unsigned numBits) + { + numBits += _bitPos; + _buf += numBits >> 3; + _bitPos = numBits & 7; + } + + + Z7_FORCE_INLINE + UInt32 ReadBits9(unsigned numBits) + { + const Byte *buf = _buf; + UInt32 v = ((UInt32)buf[0] << 8) | (UInt32)buf[1]; + v &= (UInt32)0xFFFF >> _bitPos; + numBits += _bitPos; + v >>= 16 - numBits; + _buf = buf + (numBits >> 3); + _bitPos = numBits & 7; + return v; + } + + Z7_FORCE_INLINE + UInt32 ReadBits_9fix(unsigned numBits) + { + const Byte *buf = _buf; + UInt32 v = ((UInt32)buf[0] << 8) | (UInt32)buf[1]; + const UInt32 mask = (1u << numBits) - 1; + numBits += _bitPos; + v >>= 16 - numBits; + _buf = buf + (numBits >> 3); + _bitPos = numBits & 7; + return v & mask; + } + +#if 1 && defined(MY_CPU_SIZEOF_POINTER) && (MY_CPU_SIZEOF_POINTER == 8) +#define Z7_RAR5_USE_64BIT +#endif + +#ifdef Z7_RAR5_USE_64BIT +#define MAX_DICT_LOG (sizeof(size_t) / 8 * 5 + 31) +#else +#define MAX_DICT_LOG 31 +#endif + +#ifdef Z7_RAR5_USE_64BIT + + Z7_FORCE_INLINE + size_t ReadBits_Big(unsigned numBits, UInt64 v) + { + const UInt64 mask = ((UInt64)1 << numBits) - 1; + numBits += _bitPos; + const Byte *buf = _buf; + // UInt64 v = GetBe64(buf); + v >>= 64 - numBits; + _buf = buf + (numBits >> 3); + _bitPos = numBits & 7; + return (size_t)(v & mask); + } + #define ReadBits_Big25 ReadBits_Big + +#else + + // (numBits <= 25) for 32-bit mode + Z7_FORCE_INLINE + size_t ReadBits_Big25(unsigned numBits, UInt32 v) + { + const UInt32 mask = ((UInt32)1 << numBits) - 1; + numBits += _bitPos; + v >>= 32 - numBits; + _buf += numBits >> 3; + _bitPos = numBits & 7; + return v & mask; + } + + // numBits != 0 + Z7_FORCE_INLINE + size_t ReadBits_Big(unsigned numBits, UInt32 v) + { + const Byte *buf = _buf; + // UInt32 v = GetBe32(buf); +#if 0 + const UInt32 mask = ((UInt32)1 << numBits) - 1; + numBits += _bitPos; + if (numBits > 32) + { + v <<= numBits - 32; + v |= (UInt32)buf[4] >> (40 - numBits); + } + else + v >>= 32 - numBits; + _buf = buf + (numBits >> 3); + _bitPos = numBits & 7; + return v & mask; +#else + v <<= _bitPos; + v |= (UInt32)buf[4] >> (8 - _bitPos); + v >>= 32 - numBits; + numBits += _bitPos; + _buf = buf + (numBits >> 3); + _bitPos = numBits & 7; + return v; +#endif + } +#endif +}; + + +static const unsigned kLookaheadSize = 16; +static const unsigned kInputBufferPadZone = kLookaheadSize; + +Z7_NO_INLINE void CBitDecoder::Prepare2() throw() { - const unsigned kSize = 16; if (_buf > _bufLim) return; @@ -29,27 +627,39 @@ void CBitDecoder::Prepare2() throw() _processedSize += (size_t)(_buf - _bufBase); _buf = _bufBase; + // we do not look ahead more than 16 bytes before limit checks. + if (!_wasFinished) { - UInt32 processed = (UInt32)(kInputBufSize - rem); - _hres = _stream->Read(_bufLim, (UInt32)processed, &processed); - _bufLim += processed; - _wasFinished = (processed == 0); - if (_hres != S_OK) + while (rem <= kLookaheadSize) { - _wasFinished = true; - // throw CInBufferException(result); + UInt32 processed = (UInt32)(kInputBufSize - rem); + // processed = 33; // for debug + _hres = _stream->Read(_bufLim, processed, &processed); + _bufLim += processed; + rem += processed; + if (processed == 0 || _hres != S_OK) + { + _wasFinished = true; + // if (_hres != S_OK) throw CInBufferException(result); + break; + } } } - rem = (size_t)(_bufLim - _buf); - _bufCheck = _buf; - if (rem < kSize) - memset(_bufLim, 0xFF, kSize - rem); + // we always fill pad zone here. + // so we don't need to call Prepare2() if (_wasFinished == true) + memset(_bufLim, 0xFF, kLookaheadSize); + + if (rem < kLookaheadSize) + { + _bufCheck = _buf; + // memset(_bufLim, 0xFF, kLookaheadSize - rem); + } else - _bufCheck = _bufLim - kSize; + _bufCheck = _bufLim - kLookaheadSize; - SetCheck2(); + SetCheck_forBlock(); } @@ -61,29 +671,79 @@ enum FilterType FILTER_ARM }; -static const size_t kWriteStep = (size_t)1 << 22; +static const size_t kWriteStep = (size_t)1 << 18; + // (size_t)1 << 22; // original-unrar + +// Original unRAR claims that maximum possible filter block size is (1 << 16) now, +// and (1 << 17) is minimum win size required to support filter. +// Original unRAR uses (1u << 18) for "extra safety and possible filter area size expansion" +// We can use any win size, but we use same (1u << 18) for compatibility +// with WinRar + +// static const unsigned kWinSize_Log_Min = 17; +static const size_t kWinSize_Min = 1u << 18; CDecoder::CDecoder(): _isSolid(false), - _solidAllowed(false), + _is_v7(false), _wasInit(false), - _dictSizeLog(0), + // _dictSizeLog(0), + _dictSize(kWinSize_Min), _window(NULL), _winPos(0), + _winSize(0), + _dictSize_forCheck(0), _lzSize(0), _lzEnd(0), _writtenFileSize(0), - _winSizeAllocated(0), + _filters(NULL), + _winSize_Allocated(0), _inputBuf(NULL) { +#if 1 + memcpy(m_LenPlusTable, k_LenPlusTable, sizeof(k_LenPlusTable)); +#endif + // printf("\nsizeof(CDecoder) == %d\n", sizeof(CDecoder)); } CDecoder::~CDecoder() { - ::MidFree(_window); - ::MidFree(_inputBuf); +#ifdef Z7_RAR5_SHOW_STAT + printf("\n%4d :", 0); + for (unsigned k = 0; k < kNumStats1; k++) + printf(" %8u", (unsigned)g_stats1[k]); + printf("\n"); + for (unsigned i = 0; i < kNumStats2; i++) + { + printf("\n%4d :", i); + for (unsigned k = 0; k < kNumStats1; k++) + printf(" %8u", (unsigned)g_stats2[k][i]); + } + printf("\n"); +#endif + +#define Z7_RAR_FREE_WINDOW ::BigFree(_window); + + Z7_RAR_FREE_WINDOW + z7_AlignedFree(_inputBuf); + z7_AlignedFree(_filters); +} + +Z7_NO_INLINE +void CDecoder::DeleteUnusedFilters() +{ + if (_numUnusedFilters != 0) + { + // printf("\nDeleteUnusedFilters _numFilters = %6u\n", _numFilters); + const unsigned n = _numFilters - _numUnusedFilters; + _numFilters = n; + memmove(_filters, _filters + _numUnusedFilters, n * sizeof(CFilter)); + _numUnusedFilters = 0; + } } + +Z7_NO_INLINE HRESULT CDecoder::WriteData(const Byte *data, size_t size) { HRESULT res = S_OK; @@ -104,107 +764,172 @@ HRESULT CDecoder::WriteData(const Byte *data, size_t size) return res; } + +#if defined(MY_CPU_SIZEOF_POINTER) \ + && ( MY_CPU_SIZEOF_POINTER == 4 \ + || MY_CPU_SIZEOF_POINTER == 8) + #define BR_CONV_USE_OPT_PC_PTR +#endif + +#ifdef BR_CONV_USE_OPT_PC_PTR +#define BR_PC_INIT(lim_back) pc -= (UInt32)(SizeT)data; +#define BR_PC_GET (pc + (UInt32)(SizeT)data) +#else +#define BR_PC_INIT(lim_back) pc += (UInt32)dataSize - (lim_back); +#define BR_PC_GET (pc - (UInt32)(SizeT)(data_lim - data)) +#endif + +#ifdef MY_CPU_LE_UNALIGN +#define Z7_RAR5_FILTER_USE_LE_UNALIGN +#endif + +#ifdef Z7_RAR5_FILTER_USE_LE_UNALIGN +#define RAR_E8_FILT(mask) \ +{ \ + for (;;) \ + { UInt32 v; \ + do { \ + v = GetUi32(data) ^ (UInt32)0xe8e8e8e8; \ + data += 4; \ + if ((v & ((UInt32)(mask) << (8 * 0))) == 0) { data -= 3; break; } \ + if ((v & ((UInt32)(mask) << (8 * 1))) == 0) { data -= 2; break; } \ + if ((v & ((UInt32)(mask) << (8 * 2))) == 0) { data -= 1; break; } } \ + while((v & ((UInt32)(mask) << (8 * 3)))); \ + if (data > data_lim) break; \ + const UInt32 offset = BR_PC_GET & (kFileSize - 1); \ + const UInt32 addr = GetUi32(data); \ + data += 4; \ + if (addr < kFileSize) \ + SetUi32(data - 4, addr - offset) \ + else if (addr > ~offset) /* if (addr > ((UInt32)0xFFFFFFFF - offset)) */ \ + SetUi32(data - 4, addr + kFileSize) \ + } \ +} +#else +#define RAR_E8_FILT(get_byte) \ +{ \ + for (;;) \ + { \ + if ((get_byte) != 0xe8) \ + if ((get_byte) != 0xe8) \ + if ((get_byte) != 0xe8) \ + if ((get_byte) != 0xe8) \ + continue; \ + { if (data > data_lim) break; \ + const UInt32 offset = BR_PC_GET & (kFileSize - 1); \ + const UInt32 addr = GetUi32(data); \ + data += 4; \ + if (addr < kFileSize) \ + SetUi32(data - 4, addr - offset) \ + else if (addr > ~offset) /* if (addr > ((UInt32)0xFFFFFFFF - offset)) */ \ + SetUi32(data - 4, addr + kFileSize) \ + } \ + } \ +} +#endif + HRESULT CDecoder::ExecuteFilter(const CFilter &f) { - bool useDest = false; - Byte *data = _filterSrc; UInt32 dataSize = f.Size; - // printf("\nType = %d offset = %9d size = %5d", f.Type, (unsigned)(f.Start - _lzFileStart), dataSize); - - switch (f.Type) + + if (f.Type == FILTER_DELTA) { - case FILTER_E8: - case FILTER_E8E9: + // static unsigned g1 = 0, g2 = 0; g1 += dataSize; + // if (g2++ % 100 == 0) printf("DELTA num %8u, size %8u MiB, channels = %2u curSize=%8u\n", g2, (g1 >> 20), f.Channels, dataSize); + _filterDst.AllocAtLeast_max((size_t)dataSize, k_Filter_BlockSize_MAX); + if (!_filterDst.IsAllocated()) + return E_OUTOFMEMORY; + + Byte *dest = _filterDst; + const unsigned numChannels = f.Channels; + unsigned curChannel = 0; + do { - // printf(" FILTER_E8"); - if (dataSize > 4) - { - dataSize -= 4; - const UInt32 fileOffset = (UInt32)(f.Start - _lzFileStart); - - const UInt32 kFileSize = (UInt32)1 << 24; - const Byte cmpMask = (Byte)(f.Type == FILTER_E8 ? 0xFF : 0xFE); - - for (UInt32 curPos = 0; curPos < dataSize;) - { - curPos++; - if (((*data++) & cmpMask) == 0xE8) - { - const UInt32 offset = (curPos + fileOffset) & (kFileSize - 1); - const UInt32 addr = GetUi32(data); - - if (addr < kFileSize) - { - SetUi32(data, addr - offset) - } - else if (addr > ((UInt32)0xFFFFFFFF - offset)) // (addr > ~(offset)) - { - SetUi32(data, addr + kFileSize) - } - - data += 4; - curPos += 4; - } - } - } - break; + Byte prevByte = 0; + Byte *dest2 = dest + curChannel; + const Byte *dest_lim = dest + dataSize; + for (; dest2 < dest_lim; dest2 += numChannels) + *dest2 = (prevByte = (Byte)(prevByte - *data++)); } - - case FILTER_ARM: + while (++curChannel != numChannels); + // return WriteData(dest, dataSize); + data = dest; + } + else if (f.Type < FILTER_ARM) + { + // FILTER_E8 or FILTER_E8E9 + if (dataSize > 4) { - if (dataSize >= 4) + UInt32 pc = (UInt32)(f.Start - _lzFileStart); + const UInt32 kFileSize = (UInt32)1 << 24; + const Byte *data_lim = data + dataSize - 4; + BR_PC_INIT(4) // because (data_lim) was moved back for 4 bytes + data[dataSize] = 0xe8; + if (f.Type == FILTER_E8) { - dataSize -= 4; - const UInt32 fileOffset = (UInt32)(f.Start - _lzFileStart); - - for (UInt32 curPos = 0; curPos <= dataSize; curPos += 4) - { - Byte *d = data + curPos; - if (d[3] == 0xEB) - { - UInt32 offset = d[0] | ((UInt32)d[1] << 8) | ((UInt32)d[2] << 16); - offset -= (fileOffset + curPos) >> 2; - d[0] = (Byte)offset; - d[1] = (Byte)(offset >> 8); - d[2] = (Byte)(offset >> 16); - } - } + // static unsigned g1 = 0; g1 += dataSize; printf("\n FILTER_E8 %u", (g1 >> 20)); +#ifdef Z7_RAR5_FILTER_USE_LE_UNALIGN + RAR_E8_FILT (0xff) +#else + RAR_E8_FILT (*data++) +#endif + } + else + { + // static unsigned g1 = 0; g1 += dataSize; printf("\n FILTER_E8_E9 %u", (g1 >> 20)); +#ifdef Z7_RAR5_FILTER_USE_LE_UNALIGN + RAR_E8_FILT (0xfe) +#else + RAR_E8_FILT (*data++ & 0xfe) +#endif } - break; } - - case FILTER_DELTA: + data = _filterSrc; + } + else if (f.Type == FILTER_ARM) + { + UInt32 pc = (UInt32)(f.Start - _lzFileStart); +#if 0 + // z7_BranchConv_ARM_Dec expects that (fileOffset & 3) == 0; + // but even if (fileOffset & 3) then current code + // in z7_BranchConv_ARM_Dec works same way as unrar's code still. + z7_BranchConv_ARM_Dec(data, dataSize, pc - 8); +#else + dataSize &= ~(UInt32)3; + if (dataSize) { - // printf(" channels = %d", f.Channels); - _filterDst.AllocAtLeast(dataSize); - if (!_filterDst.IsAllocated()) - return E_OUTOFMEMORY; - - Byte *dest = _filterDst; - const UInt32 numChannels = f.Channels; - - for (UInt32 curChannel = 0; curChannel < numChannels; curChannel++) + Byte *data_lim = data + dataSize; + data_lim[3] = 0xeb; + BR_PC_INIT(0) + pc -= 4; // because (data) will point to next instruction + for (;;) // do { - Byte prevByte = 0; - for (UInt32 destPos = curChannel; destPos < dataSize; destPos += numChannels) - dest[destPos] = (prevByte = (Byte)(prevByte - *data++)); + data += 4; + if (data[-1] != 0xeb) + continue; + if (data > data_lim) + break; + { + UInt32 v = GetUi32a(data - 4) - (BR_PC_GET >> 2); + v &= 0x00ffffff; + v |= 0xeb000000; + SetUi32a(data - 4, v) + } } - useDest = true; - break; } - - default: - _unsupportedFilter = true; - memset(_filterSrc, 0, f.Size); - // return S_OK; // unrar +#endif + data = _filterSrc; } - - return WriteData(useDest ? - (const Byte *)_filterDst : - (const Byte *)_filterSrc, - f.Size); + else + { + _unsupportedFilter = true; + My_ZeroMemory(data, dataSize); + // return S_OK; // unrar + } + // return WriteData(_filterSrc, (size_t)f.Size); + return WriteData(data, (size_t)f.Size); } @@ -212,13 +937,13 @@ HRESULT CDecoder::WriteBuf() { DeleteUnusedFilters(); - for (unsigned i = 0; i < _filters.Size();) + const UInt64 lzSize = _lzSize + _winPos; + + for (unsigned i = 0; i < _numFilters;) { const CFilter &f = _filters[i]; - const UInt64 blockStart = f.Start; - - const size_t lzAvail = (size_t)(_lzSize - _lzWritten); + const size_t lzAvail = (size_t)(lzSize - _lzWritten); if (lzAvail == 0) break; @@ -228,7 +953,7 @@ HRESULT CDecoder::WriteBuf() size_t size = lzAvail; if (size > rem) size = (size_t)rem; - if (size != 0) + if (size != 0) // is it true always ? { RINOK(WriteData(_window + _winPos - lzAvail, size)) _lzWritten += size; @@ -240,7 +965,9 @@ HRESULT CDecoder::WriteBuf() size_t offset = (size_t)(_lzWritten - blockStart); if (offset == 0) { - _filterSrc.AllocAtLeast(blockSize); + _filterSrc.AllocAtLeast_max( + (size_t)blockSize + k_Filter_AfterPad_Size, + k_Filter_BlockSize_MAX + k_Filter_AfterPad_Size); if (!_filterSrc.IsAllocated()) return E_OUTOFMEMORY; } @@ -248,7 +975,7 @@ HRESULT CDecoder::WriteBuf() const size_t blockRem = (size_t)blockSize - offset; size_t size = lzAvail; if (size > blockRem) - size = blockRem; + size = blockRem; memcpy(_filterSrc + offset, _window + _winPos - lzAvail, size); _lzWritten += size; offset += size; @@ -261,22 +988,28 @@ HRESULT CDecoder::WriteBuf() DeleteUnusedFilters(); - if (!_filters.IsEmpty()) + if (_numFilters) return S_OK; - const size_t lzAvail = (size_t)(_lzSize - _lzWritten); + const size_t lzAvail = (size_t)(lzSize - _lzWritten); RINOK(WriteData(_window + _winPos - lzAvail, lzAvail)) _lzWritten += lzAvail; return S_OK; } +Z7_NO_INLINE static UInt32 ReadUInt32(CBitDecoder &bi) { - const unsigned numBytes = bi.ReadBits9fix(2) + 1; + const unsigned numBits = (unsigned)bi.ReadBits_9fix(2) * 8 + 8; UInt32 v = 0; - for (unsigned i = 0; i < numBytes; i++) - v += ((UInt32)bi.ReadBits9fix(8) << (i * 8)); + unsigned i = 0; + do + { + v += (UInt32)bi.ReadBits_9fix(8) << i; + i += 8; + } + while (i != numBits); return v; } @@ -287,11 +1020,11 @@ HRESULT CDecoder::AddFilter(CBitDecoder &_bitStream) { DeleteUnusedFilters(); - if (_filters.Size() >= MAX_UNPACK_FILTERS) + if (_numFilters >= MAX_UNPACK_FILTERS) { RINOK(WriteBuf()) DeleteUnusedFilters(); - if (_filters.Size() >= MAX_UNPACK_FILTERS) + if (_numFilters >= MAX_UNPACK_FILTERS) { _unsupportedFilter = true; InitFilters(); @@ -304,17 +1037,24 @@ HRESULT CDecoder::AddFilter(CBitDecoder &_bitStream) const UInt32 blockStart = ReadUInt32(_bitStream); f.Size = ReadUInt32(_bitStream); - if (f.Size > ((UInt32)1 << 22)) + if (f.Size > k_Filter_BlockSize_MAX) { _unsupportedFilter = true; f.Size = 0; // unrar 5.5.5 } - f.Type = (Byte)_bitStream.ReadBits9fix(3); + f.Type = (Byte)_bitStream.ReadBits_9fix(3); f.Channels = 0; if (f.Type == FILTER_DELTA) - f.Channels = (Byte)(_bitStream.ReadBits9fix(5) + 1); - f.Start = _lzSize + blockStart; + f.Channels = (Byte)(_bitStream.ReadBits_9fix(5) + 1); + f.Start = _lzSize + _winPos + blockStart; + +#if 0 + static unsigned z_cnt = 0; if (z_cnt++ % 100 == 0) + printf ("\nFilter %7u : %4u : %8p, st=%8x, size=%8x, type=%u ch=%2u", + z_cnt, (unsigned)_filters.Size(), (void *)(size_t)(_lzSize + _winPos), + (unsigned)blockStart, (unsigned)f.Size, (unsigned)f.Type, (unsigned)f.Channels); +#endif if (f.Start < _filterEnd) _unsupportedFilter = true; @@ -322,7 +1062,17 @@ HRESULT CDecoder::AddFilter(CBitDecoder &_bitStream) { _filterEnd = f.Start + f.Size; if (f.Size != 0) - _filters.Add(f); + { + if (!_filters) + { + _filters = (CFilter *)z7_AlignedAlloc(MAX_UNPACK_FILTERS * sizeof(CFilter)); + if (!_filters) + return E_OUTOFMEMORY; + } + // printf("\n_numFilters = %6u\n", _numFilters); + const unsigned i = _numFilters++; + _filters[i] = f; + } } return S_OK; @@ -331,126 +1081,184 @@ HRESULT CDecoder::AddFilter(CBitDecoder &_bitStream) #define RIF(x) { if (!(x)) return S_FALSE; } +#if 1 +#define PRINT_CNT(name, skip) +#else +#define PRINT_CNT(name, skip) \ + { static unsigned g_cnt = 0; if (g_cnt++ % skip == 0) printf("\n%16s: %8u", name, g_cnt); } +#endif + HRESULT CDecoder::ReadTables(CBitDecoder &_bitStream) { if (_progress) { const UInt64 packSize = _bitStream.GetProcessedSize(); - RINOK(_progress->SetRatioInfo(&packSize, &_writtenFileSize)) + if (packSize - _progress_Pack >= (1u << 24) + || _writtenFileSize - _progress_Unpack >= (1u << 26)) + { + _progress_Pack = packSize; + _progress_Unpack = _writtenFileSize; + RINOK(_progress->SetRatioInfo(&_progress_Pack, &_writtenFileSize)) + } + // printf("\ntable read pos=%p packSize=%p _writtenFileSize = %p\n", (size_t)_winPos, (size_t)packSize, (size_t)_writtenFileSize); } - _bitStream.AlignToByte(); + // _bitStream is aligned already _bitStream.Prepare(); - { - const unsigned flags = _bitStream.ReadByteInAligned(); - unsigned checkSum = _bitStream.ReadByteInAligned(); + const unsigned flags = _bitStream.ReadByte_InAligned(); + /* ((flags & 20) == 0) in all rar archives now, + but (flags & 20) flag can be used as some decoding hint in future versions of original rar. + So we ignore that bit here. */ + unsigned checkSum = _bitStream.ReadByte_InAligned(); checkSum ^= flags; const unsigned num = (flags >> 3) & 3; - if (num == 3) + if (num >= 3) return S_FALSE; - UInt32 blockSize = _bitStream.ReadByteInAligned(); + UInt32 blockSize = _bitStream.ReadByte_InAligned(); checkSum ^= blockSize; - if (num != 0) { - unsigned b = _bitStream.ReadByteInAligned(); - checkSum ^= b; - blockSize += (UInt32)b << 8; + { + const unsigned b = _bitStream.ReadByte_InAligned(); + checkSum ^= b; + blockSize += (UInt32)b << 8; + } if (num > 1) { - b = _bitStream.ReadByteInAligned(); + const unsigned b = _bitStream.ReadByte_InAligned(); checkSum ^= b; blockSize += (UInt32)b << 16; } } - if (checkSum != 0x5A) return S_FALSE; - unsigned blockSizeBits7 = (flags & 7) + 1; - blockSize += (blockSizeBits7 >> 3); + blockSize += (UInt32)(blockSizeBits7 >> 3); if (blockSize == 0) + { + // it's error in data stream + // but original-unrar ignores that error + _bitStream._minorError = true; +#if 1 + // we ignore that error as original-unrar: + blockSizeBits7 = 0; + blockSize = 1; +#else + // we can stop decoding: return S_FALSE; +#endif + } blockSize--; blockSizeBits7 &= 7; - - _bitStream._blockEndBits7 = (Byte)blockSizeBits7; + PRINT_CNT("Blocks", 100) + /* + { + static unsigned g_prev = 0; + static unsigned g_cnt = 0; + unsigned proc = unsigned(_winPos); + if (g_cnt++ % 100 == 0) printf(" c_size = %8u ", blockSize); + if (g_cnt++ % 100 == 1) printf(" unp_size = %8u", proc - g_prev); + g_prev = proc; + } + */ + _bitStream._blockEndBits7 = blockSizeBits7; _bitStream._blockEnd = _bitStream.GetProcessedSize_Round() + blockSize; - - _bitStream.SetCheck2(); - + _bitStream.SetCheck_forBlock(); _isLastBlock = ((flags & 0x40) != 0); - if ((flags & 0x80) == 0) { if (!_tableWasFilled) - if (blockSize != 0 || blockSizeBits7 != 0) + // if (blockSize != 0 || blockSizeBits7 != 0) + if (blockSize + blockSizeBits7 != 0) return S_FALSE; return S_OK; } - _tableWasFilled = false; } + PRINT_CNT("Tables", 100); + + const unsigned kLevelTableSize = 20; + const unsigned k_NumHufTableBits_Level = 6; + NHuffman::CDecoder256 m_LevelDecoder; + const unsigned kTablesSizesSum_MAX = kMainTableSize + kDistTableSize_MAX + kAlignTableSize + kLenTableSize; + Byte lens[kTablesSizesSum_MAX]; { - Byte lens2[kLevelTableSize]; - - for (unsigned i = 0; i < kLevelTableSize;) + // (kLevelTableSize + 16 < kTablesSizesSum). So we use lens[] array for (Level) table + // Byte lens2[kLevelTableSize + 16]; + unsigned i = 0; + do { - _bitStream.Prepare(); - const unsigned len = (unsigned)_bitStream.ReadBits9fix(4); + if (_bitStream._buf >= _bitStream._bufCheck_Block) + { + _bitStream.Prepare(); + if (_bitStream.IsBlockOverRead()) + return S_FALSE; + } + const unsigned len = (unsigned)_bitStream.ReadBits_9fix(4); if (len == 15) { - unsigned num = (unsigned)_bitStream.ReadBits9fix(4); + unsigned num = (unsigned)_bitStream.ReadBits_9fix(4); if (num != 0) { num += 2; num += i; + // we are allowed to overwrite to lens[] for extra 16 bytes after kLevelTableSize +#if 0 if (num > kLevelTableSize) + { + // we ignore this error as original-unrar num = kLevelTableSize; + // return S_FALSE; + } +#endif do - lens2[i++] = 0; + lens[i++] = 0; while (i < num); continue; } } - lens2[i++] = (Byte)len; + lens[i++] = (Byte)len; } - + while (i < kLevelTableSize); if (_bitStream.IsBlockOverRead()) return S_FALSE; - - RIF(m_LevelDecoder.Build(lens2)) + RIF(m_LevelDecoder.Build(lens, NHuffman::k_BuildMode_Full)) } - - Byte lens[kTablesSizesSum]; + unsigned i = 0; - + const unsigned tableSize = _is_v7 ? + kTablesSizesSum_MAX : + kTablesSizesSum_MAX - kExtraDistSymbols_v7; do { - if (_bitStream._buf >= _bitStream._bufCheck2) + if (_bitStream._buf >= _bitStream._bufCheck_Block) { - if (_bitStream._buf >= _bitStream._bufCheck) - _bitStream.Prepare(); + // if (_bitStream._buf >= _bitStream._bufCheck) + _bitStream.Prepare(); if (_bitStream.IsBlockOverRead()) return S_FALSE; } - - const UInt32 sym = m_LevelDecoder.Decode(&_bitStream); - + const unsigned sym = m_LevelDecoder.DecodeFull(&_bitStream); if (sym < 16) lens[i++] = (Byte)sym; +#if 0 else if (sym > kLevelTableSize) return S_FALSE; +#endif else { - unsigned num = ((sym - 16) & 1) * 4; + unsigned num = ((sym /* - 16 */) & 1) * 4; num += num + 3 + (unsigned)_bitStream.ReadBits9(num + 3); num += i; - if (num > kTablesSizesSum) - num = kTablesSizesSum; - Byte v = 0; + if (num > tableSize) + { + // we ignore this error as original-unrar + num = tableSize; + // return S_FALSE; + } + unsigned v = 0; if (sym < 16 + 2) { if (i == 0) @@ -458,27 +1266,63 @@ HRESULT CDecoder::ReadTables(CBitDecoder &_bitStream) v = lens[(size_t)i - 1]; } do - lens[i++] = v; + lens[i++] = (Byte)v; while (i < num); } } - while (i < kTablesSizesSum); + while (i < tableSize); if (_bitStream.IsBlockOverRead()) return S_FALSE; if (_bitStream.InputEofError()) return S_FALSE; - RIF(m_MainDecoder.Build(&lens[0])) - RIF(m_DistDecoder.Build(&lens[kMainTableSize])) - RIF(m_AlignDecoder.Build(&lens[kMainTableSize + kDistTableSize])) - RIF(m_LenDecoder.Build(&lens[kMainTableSize + kDistTableSize + kAlignTableSize])) + /* We suppose that original-rar encoder can create only two cases for Huffman: + 1) Empty Huffman tree (if num_used_symbols == 0) + 2) Full Huffman tree (if num_used_symbols != 0) + Usually the block contains at least one symbol for m_MainDecoder. + So original-rar-encoder creates full Huffman tree for m_MainDecoder. + But we suppose that (num_used_symbols == 0) is possible for m_MainDecoder, + because file must be finished with (_isLastBlock) flag, + even if there are no symbols in m_MainDecoder. + So we use k_BuildMode_Full_or_Empty for m_MainDecoder. + */ + const NHuffman::enum_BuildMode buildMode = NHuffman:: + k_BuildMode_Full_or_Empty; // strict check + // k_BuildMode_Partial; // non-strict check (ignore errors) + + RIF(m_MainDecoder.Build(&lens[0], buildMode)) + if (!_is_v7) + { +#if 1 + /* we use this manual loop to avoid compiler BUG. + GCC 4.9.2 compiler has BUG with overlapping memmove() to right in local array. */ + Byte *dest = lens + kMainTableSize + kDistTableSize_v6 + + kAlignTableSize + kLenTableSize - 1; + unsigned num = kAlignTableSize + kLenTableSize; + do + { + dest[kExtraDistSymbols_v7] = dest[0]; + dest--; + } + while (--num); +#else + memmove(lens + kMainTableSize + kDistTableSize_v6 + kExtraDistSymbols_v7, + lens + kMainTableSize + kDistTableSize_v6, + kAlignTableSize + kLenTableSize); +#endif + memset(lens + kMainTableSize + kDistTableSize_v6, 0, kExtraDistSymbols_v7); + } + + RIF(m_DistDecoder.Build(&lens[kMainTableSize], buildMode)) + RIF( m_LenDecoder.Build(&lens[kMainTableSize + + kDistTableSize_MAX + kAlignTableSize], buildMode)) _useAlignBits = false; - // _useAlignBits = true; for (i = 0; i < kAlignTableSize; i++) - if (lens[kMainTableSize + kDistTableSize + (size_t)i] != kNumAlignBits) + if (lens[kMainTableSize + kDistTableSize_MAX + (size_t)i] != kNumAlignBits) { + RIF(m_AlignDecoder.Build(&lens[kMainTableSize + kDistTableSize_MAX], buildMode)) _useAlignBits = true; break; } @@ -487,139 +1331,108 @@ HRESULT CDecoder::ReadTables(CBitDecoder &_bitStream) return S_OK; } +static inline CLenType SlotToLen(CBitDecoder &_bitStream, CLenType slot) +{ + const unsigned numBits = ((unsigned)slot >> 2) - 1; + return ((4 | (slot & 3)) << numBits) + (CLenType)_bitStream.ReadBits9(numBits); +} + + +static const unsigned kSymbolRep = 258; +static const unsigned kMaxMatchLen = 0x1001 + 3; -static inline unsigned SlotToLen(CBitDecoder &_bitStream, unsigned slot) +enum enum_exit_type { - if (slot < 8) - return slot + 2; - const unsigned numBits = (slot >> 2) - 1; - return 2 + ((4 | (slot & 3)) << numBits) + _bitStream.ReadBits9(numBits); + Z7_RAR_EXIT_TYPE_NONE, + Z7_RAR_EXIT_TYPE_ADD_FILTER +}; + + +#define LZ_RESTORE \ +{ \ + _reps[0] = rep0; \ + _winPos = (size_t)(winPos - _window); \ + _buf_Res = _bitStream._buf; \ + _bitPos_Res = _bitStream._bitPos; \ } +#define LZ_LOOP_BREAK_OK { break; } +// #define LZ_LOOP_BREAK_ERROR { _lzError = LZ_ERROR_TYPE_SYM; break; } +// #define LZ_LOOP_BREAK_ERROR { LZ_RESTORE; return S_FALSE; } +#define LZ_LOOP_BREAK_ERROR { goto decode_error; } +// goto decode_error; } +// #define LZ_LOOP_BREAK_ERROR { break; } -static const UInt32 kSymbolRep = 258; -// static const unsigned kMaxMatchLen = 0x1001 + 3; +#define Z7_RAR_HUFF_DECODE_CHECK_break(sym, huf, kNumTableBits, bitStream) \ + Z7_HUFF_DECODE_CHECK(sym, huf, kNumHufBits, kNumTableBits, bitStream, { LZ_LOOP_BREAK_ERROR }) -HRESULT CDecoder::DecodeLZ() + +HRESULT CDecoder::DecodeLZ2(const CBitDecoder &bitStream) throw() { - CBitDecoder _bitStream; - _bitStream._stream = _inStream; - _bitStream._bufBase = _inputBuf; - _bitStream.Init(); +#if 0 + Byte k_LenPlusTable_LOC[DICT_SIZE_BITS_MAX]; + memcpy(k_LenPlusTable_LOC, k_LenPlusTable, sizeof(k_LenPlusTable)); +#endif - UInt32 rep0 = _reps[0]; + PRINT_CNT("DecodeLZ2", 2000); - UInt32 remLen = 0; + CBitDecoder _bitStream; + _bitStream.CopyFrom(bitStream); + // _bitStream._stream = _inStream; + // _bitStream._bufBase = _inputBuf; + // _bitStream.Init(); + + // _reps[*] can be larger than _winSize, if _winSize was reduced in solid stream. + size_t rep0 = _reps[0]; + // size_t rep1 = _reps[1]; + // Byte *win = _window; + Byte *winPos = _window + _winPos; + const Byte *limit = _window + _limit; + _exitType = Z7_RAR_EXIT_TYPE_NONE; - size_t limit; - { - size_t rem = _winSize - _winPos; - if (rem > kWriteStep) - rem = kWriteStep; - limit = _winPos + rem; - } - for (;;) { - if (_winPos >= limit) - { - RINOK(WriteBuf()) - if (_unpackSize_Defined && _writtenFileSize > _unpackSize) - break; // return S_FALSE; - - { - size_t rem = _winSize - _winPos; - - if (rem == 0) - { - _winPos = 0; - rem = _winSize; - } - if (rem > kWriteStep) - rem = kWriteStep; - limit = _winPos + rem; - } - - if (remLen != 0) - { - size_t winPos = _winPos; - const size_t winMask = _winMask; - size_t pos = (winPos - (size_t)rep0 - 1) & winMask; - - Byte *win = _window; - do - { - if (winPos >= limit) - break; - win[winPos] = win[pos]; - winPos++; - pos = (pos + 1) & winMask; - } - while (--remLen != 0); - - _lzSize += winPos - _winPos; - _winPos = winPos; - continue; - } - } - - if (_bitStream._buf >= _bitStream._bufCheck2) + if (winPos >= limit) + LZ_LOOP_BREAK_OK + // (winPos < limit) + if (_bitStream._buf >= _bitStream._bufCheck_Block) { if (_bitStream.InputEofError()) - break; // return S_FALSE; + LZ_LOOP_BREAK_OK if (_bitStream._buf >= _bitStream._bufCheck) - _bitStream.Prepare2(); - - const UInt64 processed = _bitStream.GetProcessedSize_Round(); - if (processed >= _bitStream._blockEnd) { - if (processed > _bitStream._blockEnd) - break; // return S_FALSE; - { - const unsigned bits7 = _bitStream.GetProcessedBits7(); - if (bits7 > _bitStream._blockEndBits7) - break; // return S_FALSE; - if (bits7 == _bitStream._blockEndBits7) - { - if (_isLastBlock) - { - _reps[0] = rep0; - - if (_bitStream.InputEofError()) - break; - - /* - // packSize can be 15 bytes larger for encrypted archive - if (_packSize_Defined && _packSize < _bitStream.GetProcessedSize()) - break; - */ - - return _bitStream._hres; - // break; - } - RINOK(ReadTables(_bitStream)) - continue; - } - } + if (!_bitStream._wasFinished) + LZ_LOOP_BREAK_OK + // _bitStream._wasFinished == true + // we don't need Prepare() here, because all data was read + // and PadZone (16 bytes) after data was filled. } - + const UInt64 processed = _bitStream.GetProcessedSize_Round(); + // some cases are error, but the caller will process such error cases. + if (processed >= _bitStream._blockEnd && + (processed > _bitStream._blockEnd + || _bitStream.GetProcessedBits7() >= _bitStream._blockEndBits7)) + LZ_LOOP_BREAK_OK // that check is not required, but it can help, if there is BUG in another code if (!_tableWasFilled) - break; // return S_FALSE; + LZ_LOOP_BREAK_ERROR } - - const UInt32 sym = m_MainDecoder.Decode(&_bitStream); + +#if 0 + const unsigned sym = m_MainDecoder.Decode(&_bitStream); +#else + unsigned sym; + Z7_RAR_HUFF_DECODE_CHECK_break(sym, &m_MainDecoder, k_NumHufTableBits_Main, &_bitStream) +#endif if (sym < 256) { - size_t winPos = _winPos; - _window[winPos] = (Byte)sym; - _winPos = winPos + 1; - _lzSize++; + *winPos++ = (Byte)sym; + // _lzSize++; continue; } - UInt32 len; + CLenType len; if (sym < kSymbolRep + kNumReps) { @@ -627,130 +1440,343 @@ HRESULT CDecoder::DecodeLZ() { if (sym != kSymbolRep) { - UInt32 dist; - if (sym == kSymbolRep + 1) - dist = _reps[1]; - else - { - if (sym == kSymbolRep + 2) - dist = _reps[2]; - else - { - dist = _reps[3]; - _reps[3] = _reps[2]; - } - _reps[2] = _reps[1]; - } + size_t dist = _reps[1]; _reps[1] = rep0; rep0 = dist; + if (sym >= kSymbolRep + 2) + { + #if 1 + rep0 = _reps[(size_t)sym - kSymbolRep]; + _reps[(size_t)sym - kSymbolRep] = _reps[2]; + _reps[2] = dist; + #else + if (sym != kSymbolRep + 2) + { + rep0 = _reps[3]; + _reps[3] = _reps[2]; + _reps[2] = dist; + } + else + { + rep0 = _reps[2]; + _reps[2] = dist; + } + #endif + } } - - const UInt32 sym2 = m_LenDecoder.Decode(&_bitStream); - if (sym2 >= kLenTableSize) - break; // return S_FALSE; - len = SlotToLen(_bitStream, sym2); +#if 0 + len = m_LenDecoder.Decode(&_bitStream); + if (len >= kLenTableSize) + LZ_LOOP_BREAK_ERROR +#else + Z7_RAR_HUFF_DECODE_CHECK_break(len, &m_LenDecoder, k_NumHufTableBits_Len, &_bitStream) +#endif + if (len >= 8) + len = SlotToLen(_bitStream, len); + len += 2; + // _lastLen = (UInt32)len; } - else + else if (sym != 256) { - if (sym == 256) + len = (CLenType)_lastLen; + if (len == 0) { - RINOK(AddFilter(_bitStream)) + // we ignore (_lastLen == 0) case, like original-unrar. + // that case can mean error in stream. + // lzError = true; + // return S_FALSE; continue; } - else // if (sym == 257) - { - len = _lastLen; - // if (len = 0), we ignore that symbol, like original unRAR code, but it can mean error in stream. - // if (len == 0) return S_FALSE; - if (len == 0) - continue; - } + } + else + { + _exitType = Z7_RAR_EXIT_TYPE_ADD_FILTER; + LZ_LOOP_BREAK_OK } } +#if 0 else if (sym >= kMainTableSize) - break; // return S_FALSE; + LZ_LOOP_BREAK_ERROR +#endif else { _reps[3] = _reps[2]; _reps[2] = _reps[1]; _reps[1] = rep0; - len = SlotToLen(_bitStream, sym - (kSymbolRep + kNumReps)); - - rep0 = m_DistDecoder.Decode(&_bitStream); + len = sym - (kSymbolRep + kNumReps); + if (len >= 8) + len = SlotToLen(_bitStream, len); + len += 2; + // _lastLen = (UInt32)len; +#if 0 + rep0 = (UInt32)m_DistDecoder.Decode(&_bitStream); +#else + Z7_RAR_HUFF_DECODE_CHECK_break(rep0, &m_DistDecoder, k_NumHufTableBits_Dist, &_bitStream) +#endif + if (rep0 >= 4) { - if (rep0 >= _numCorrectDistSymbols) - break; // return S_FALSE; - const unsigned numBits = (rep0 >> 1) - 1; +#if 0 + if (rep0 >= kDistTableSize_MAX) + LZ_LOOP_BREAK_ERROR +#endif + const unsigned numBits = ((unsigned)rep0 - 2) >> 1; rep0 = (2 | (rep0 & 1)) << numBits; - + + const Byte *buf = _bitStream._buf; +#ifdef Z7_RAR5_USE_64BIT + const UInt64 v = GetBe64(buf); +#else + const UInt32 v = GetBe32(buf); +#endif + + // _lastLen = (UInt32)len; if (numBits < kNumAlignBits) - rep0 += _bitStream.ReadBits9(numBits); + { + rep0 += // _bitStream.ReadBits9(numBits); + _bitStream.ReadBits_Big25(numBits, v); + } else { - len += (numBits >= 7); - len += (numBits >= 12); - len += (numBits >= 17); - + #if !defined(MY_CPU_AMD64) + len += k_LenPlusTable[numBits]; + #elif 0 + len += k_LenPlusTable_LOC[numBits]; + #elif 1 + len += m_LenPlusTable[numBits]; + #elif 1 && defined(MY_CPU_64BIT) && defined(MY_CPU_AMD64) + // len += (unsigned)((UInt64)0xfffffffeaa554000 >> (numBits * 2)) & 3; + len += (unsigned)((UInt64)0xfffffffffeaa5540 >> (numBits * 2 - 8)) & 3; + #elif 1 + len += 3; + len -= (unsigned)(numBits - 7) >> (sizeof(unsigned) * 8 - 1); + len -= (unsigned)(numBits - 12) >> (sizeof(unsigned) * 8 - 1); + len -= (unsigned)(numBits - 17) >> (sizeof(unsigned) * 8 - 1); + #elif 1 + len += 3; + len -= (0x155aabf >> (numBits - 4) >> (numBits - 4)) & 3; + #elif 1 + len += (numBits >= 7); + len += (numBits >= 12); + len += (numBits >= 17); + #endif + // _lastLen = (UInt32)len; if (_useAlignBits) { // if (numBits > kNumAlignBits) - rep0 += (_bitStream.ReadBits32(numBits - kNumAlignBits) << kNumAlignBits); - const UInt32 a = m_AlignDecoder.Decode(&_bitStream); + rep0 += (_bitStream.ReadBits_Big25(numBits - kNumAlignBits, v) << kNumAlignBits); +#if 0 + const unsigned a = m_AlignDecoder.Decode(&_bitStream); if (a >= kAlignTableSize) - break; // return S_FALSE; + LZ_LOOP_BREAK_ERROR +#else + unsigned a; + Z7_RAR_HUFF_DECODE_CHECK_break(a, &m_AlignDecoder, k_NumHufTableBits_Align, &_bitStream) +#endif rep0 += a; } else - rep0 += _bitStream.ReadBits32(numBits); + rep0 += _bitStream.ReadBits_Big(numBits, v); +#ifndef Z7_RAR5_USE_64BIT + if (numBits >= 30) // we don't want 32-bit overflow case + rep0 = (size_t)0 - 1 - 1; +#endif } } + rep0++; } - _lastLen = len; - - if (rep0 >= _lzSize) - _lzError = true; - { - UInt32 lenCur = len; - size_t winPos = _winPos; - size_t pos = (winPos - (size_t)rep0 - 1) & _winMask; + _lastLen = (UInt32)len; + // len != 0 + +#ifdef Z7_RAR5_SHOW_STAT { - const size_t rem = limit - winPos; - // size_t rem = _winSize - winPos; + size_t index = rep0; + if (index >= kNumStats1) + index = kNumStats1 - 1; + g_stats1[index]++; + g_stats2[index][len]++; + } +#endif - if (lenCur > rem) + Byte *dest = winPos; + winPos += len; + if (rep0 <= _dictSize_forCheck) + { + const Byte *src; + const size_t winPos_temp = (size_t)(dest - _window); + if (rep0 > winPos_temp) { - lenCur = (UInt32)rem; - remLen = len - lenCur; + if (_lzSize == 0) + goto error_dist; + size_t back = rep0 - winPos_temp; + // STAT_INC(g_NumOver) + src = dest + (_winSize - rep0); + if (back < len) + { + // len -= (CLenType)back; + Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE + do + *dest++ = *src++; + while (--back); + src = dest - rep0; + } } + else + src = dest - rep0; + CopyMatch(rep0, dest, src, winPos); + continue; } - - Byte *win = _window; - _lzSize += lenCur; - _winPos = winPos + lenCur; - if (_winSize - pos >= lenCur) + +error_dist: + // LZ_LOOP_BREAK_ERROR; + _lzError = LZ_ERROR_TYPE_DIST; + do + *dest++ = 0; + while (dest < winPos); + continue; + } + } + + LZ_RESTORE + return S_OK; + +#if 1 +decode_error: + /* + if (_bitStream._hres != S_OK) + return _bitStream._hres; + */ + LZ_RESTORE + return S_FALSE; +#endif +} + + + +HRESULT CDecoder::DecodeLZ() +{ + CBitDecoder _bitStream; + _bitStream._stream = _inStream; + _bitStream._bufBase = _inputBuf; + _bitStream.Init(); + + // _reps[*] can be larger than _winSize, if _winSize was reduced in solid stream. + size_t winPos = _winPos; + Byte *win = _window; + size_t limit; + { + size_t rem = _winSize - winPos; + if (rem > kWriteStep) + rem = kWriteStep; + limit = winPos + rem; + } + + for (;;) + { + if (winPos >= limit) + { + _winPos = winPos < _winSize ? winPos : _winSize; + RINOK(WriteBuf()) + if (_unpackSize_Defined && _writtenFileSize > _unpackSize) + break; // return S_FALSE; + const size_t wp = _winPos; + size_t rem = _winSize - wp; + if (rem == 0) { - const Byte *src = win + pos; - Byte *dest = win + winPos; - do - *dest++ = *src++; - while (--lenCur != 0); + _lzSize += wp; + winPos -= wp; + // (winPos < kMaxMatchLen < _winSize) + // so memmove is not required here + if (winPos) + memcpy(win, win + _winSize, winPos); + limit = _winSize; + if (limit >= kWriteStep) + { + limit = kWriteStep; + continue; + } + rem = _winSize - winPos; } - else + if (rem > kWriteStep) + rem = kWriteStep; + limit = winPos + rem; + continue; + } + + // (winPos < limit) + + if (_bitStream._buf >= _bitStream._bufCheck_Block) + { + _winPos = winPos; + if (_bitStream.InputEofError()) + break; // return S_FALSE; + _bitStream.Prepare(); + + const UInt64 processed = _bitStream.GetProcessedSize_Round(); + if (processed >= _bitStream._blockEnd) { - do + if (processed > _bitStream._blockEnd) + break; // return S_FALSE; { - win[winPos] = win[pos]; - winPos++; - pos = (pos + 1) & _winMask; + const unsigned bits7 = _bitStream.GetProcessedBits7(); + if (bits7 >= _bitStream._blockEndBits7) + { + if (bits7 > _bitStream._blockEndBits7) + { +#if 1 + // we ignore thar error as original unrar + _bitStream._minorError = true; +#else + break; // return S_FALSE; +#endif + } + _bitStream.AlignToByte(); + // if (!_bitStream.AlignToByte()) break; + if (_isLastBlock) + { + if (_bitStream.InputEofError()) + break; + /* + // packSize can be 15 bytes larger for encrypted archive + if (_packSize_Defined && _packSize < _bitStream.GetProcessedSize()) + break; + */ + if (_bitStream._minorError) + return S_FALSE; + return _bitStream._hres; + // break; + } + RINOK(ReadTables(_bitStream)) + continue; + } } - while (--lenCur != 0); } + + // end of block was not reached. + // so we must decode more symbols + // that check is not required, but it can help, if there is BUG in another code + if (!_tableWasFilled) + break; // return S_FALSE; + } + + _limit = limit; + _winPos = winPos; + RINOK(DecodeLZ2(_bitStream)) + _bitStream._buf = _buf_Res; + _bitStream._bitPos = _bitPos_Res; + + winPos = _winPos; + if (_exitType == Z7_RAR_EXIT_TYPE_ADD_FILTER) + { + RINOK(AddFilter(_bitStream)) + continue; } } + + _winPos = winPos; if (_bitStream._hres != S_OK) return _bitStream._hres; @@ -759,221 +1785,238 @@ HRESULT CDecoder::DecodeLZ() } + HRESULT CDecoder::CodeReal() { _unsupportedFilter = false; - _lzError = false; _writeError = false; - + /* if (!_isSolid || !_wasInit) { - size_t clearSize = _winSize; - if (_lzSize < _winSize) - clearSize = (size_t)_lzSize; - memset(_window, 0, clearSize); - _wasInit = true; - _lzSize = 0; + // _lzSize = 0; _lzWritten = 0; _winPos = 0; - for (unsigned i = 0; i < kNumReps; i++) - _reps[i] = (UInt32)0 - 1; - + _reps[i] = (size_t)0 - 1; _lastLen = 0; _tableWasFilled = false; } - + */ _isLastBlock = false; InitFilters(); _filterEnd = 0; _writtenFileSize = 0; - - _lzFileStart = _lzSize; - _lzWritten = _lzSize; + const UInt64 lzSize = _lzSize + _winPos; + _lzFileStart = lzSize; + _lzWritten = lzSize; HRESULT res = DecodeLZ(); HRESULT res2 = S_OK; if (!_writeError && res != E_OUTOFMEMORY) res2 = WriteBuf(); - /* if (res == S_OK) if (InputEofError()) res = S_FALSE; */ - if (res == S_OK) { - _solidAllowed = true; + // _solidAllowed = true; res = res2; } - if (res == S_OK && _unpackSize_Defined && _writtenFileSize != _unpackSize) return S_FALSE; return res; } -// Original unRAR claims that maximum possible filter block size is (1 << 16) now, -// and (1 << 17) is minimum win size required to support filter. -// Original unRAR uses (1 << 18) for "extra safety and possible filter area size expansion" -// We can use any win size. - -static const unsigned kWinSize_Log_Min = 17; Z7_COM7F_IMF(CDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 * /* inSize */, const UInt64 *outSize, ICompressProgressInfo *progress)) { - try + _lzError = LZ_ERROR_TYPE_NO; +/* + if file is soild, but decoding of previous file was not finished, + we still try to decode new file. + We need correct huffman table at starting block. + And rar encoder probably writes huffman table at start block, if file is big. + So we have good chance to get correct huffman table in some file after corruption. + Also we try to recover window by filling zeros, if previous file + was decoded to smaller size than required. + But if filling size is big, we do full reset of window instead. +*/ + #define Z7_RAR_RECOVER_SOLID_LIMIT (1 << 20) + // #define Z7_RAR_RECOVER_SOLID_LIMIT 0 // do not fill zeros { - if (_isSolid && !_solidAllowed) - return S_FALSE; - _solidAllowed = false; - - if (_dictSizeLog >= sizeof(size_t) * 8) - return E_NOTIMPL; - - if (!_isSolid) + // if (_winPos > 100) _winPos -= 100; // for debug: corruption + const UInt64 lzSize = _lzSize + _winPos; + if (!_isSolid || !_wasInit + || (lzSize < _lzEnd +#if Z7_RAR_RECOVER_SOLID_LIMIT != 0 + && lzSize + Z7_RAR_RECOVER_SOLID_LIMIT < _lzEnd +#endif + )) + { + if (_isSolid) + _lzError = LZ_ERROR_TYPE_HEADER; _lzEnd = 0; - else + _lzSize = 0; + _lzWritten = 0; + _winPos = 0; + for (unsigned i = 0; i < kNumReps; i++) + _reps[i] = (size_t)0 - 1; + _lastLen = 0; + _tableWasFilled = false; + _wasInit = true; + } +#if Z7_RAR_RECOVER_SOLID_LIMIT != 0 + else if (lzSize < _lzEnd) { - if (_lzSize < _lzEnd) +#if 0 + return S_FALSE; +#else + // we can report that recovering was made: + // _lzError = LZ_ERROR_TYPE_HEADER; + // We write zeros to area after corruption: + if (_window) { - if (_window) + UInt64 rem = _lzEnd - lzSize; + const size_t ws = _winSize; + if (rem >= ws) { - UInt64 rem = _lzEnd - _lzSize; - if (rem >= _winSize) - memset(_window, 0, _winSize); - else + My_ZeroMemory(_window, ws); + _lzSize = ws; + _winPos = 0; + } + else + { + const size_t cur = ws - _winPos; + if (cur <= rem) { - const size_t pos = (size_t)_lzSize & _winSize; - size_t rem2 = _winSize - pos; - if (rem2 > rem) - rem2 = (size_t)rem; - memset(_window + pos, 0, rem2); - rem -= rem2; - memset(_window, 0, (size_t)rem); + rem -= cur; + My_ZeroMemory(_window + _winPos, cur); + _lzSize += _winPos; + _winPos = 0; } + My_ZeroMemory(_window + _winPos, (size_t)rem); + _winPos += (size_t)rem; } - _lzEnd &= ((((UInt64)1) << 33) - 1); - _lzSize = _lzEnd; - _winPos = (size_t)(_lzSize & _winSize); } - _lzEnd = _lzSize; + // else return S_FALSE; +#endif } +#endif + } + + // we don't want _lzSize overflow + if (_lzSize >= DICT_SIZE_MAX) + _lzSize = DICT_SIZE_MAX; + _lzEnd = _lzSize + _winPos; + // _lzSize <= DICT_SIZE_MAX + // _lzEnd <= DICT_SIZE_MAX * 2 - size_t newSize; + size_t newSize = _dictSize; + if (newSize < kWinSize_Min) + newSize = kWinSize_Min; + + _unpackSize = 0; + _unpackSize_Defined = (outSize != NULL); + if (_unpackSize_Defined) + _unpackSize = *outSize; + + if ((Int64)_unpackSize >= 0) + _lzEnd += _unpackSize; // known end after current file + else + _lzEnd = 0; // unknown end + + if (_isSolid && _window) + { + // If dictionary was decreased in solid, we use old dictionary. + if (newSize > _dictSize_forCheck) { - unsigned newSizeLog = _dictSizeLog; - if (newSizeLog < kWinSize_Log_Min) - newSizeLog = kWinSize_Log_Min; - newSize = (size_t)1 << newSizeLog; - _numCorrectDistSymbols = newSizeLog * 2; + // If dictionary was increased in solid, we don't want grow. + return S_FALSE; // E_OUTOFMEMORY } - - // If dictionary was reduced, we use allocated dictionary block - // for compatibility with original unRAR decoder. - - if (_window && newSize < _winSizeAllocated) - _winSize = _winSizeAllocated; - else if (!_window || _winSize != newSize) + // (newSize <= _winSize) + } + else + { + _dictSize_forCheck = newSize; { - if (!_isSolid) - { - ::MidFree(_window); - _window = NULL; - _winSizeAllocated = 0; - } - - Byte *win; - - { - win = (Byte *)::MidAlloc(newSize); - if (!win) - return E_OUTOFMEMORY; - memset(win, 0, newSize); - } - - if (_isSolid && _window) - { - // original unRAR claims: - // "Archiving code guarantees that win size does not grow in the same solid stream", - // but the original unRAR decoder still supports such grow case. - - Byte *winOld = _window; - const size_t oldSize = _winSize; - const size_t newMask = newSize - 1; - const size_t oldMask = _winSize - 1; - const size_t winPos = _winPos; - for (size_t i = 1; i <= oldSize; i++) - win[(winPos - i) & newMask] = winOld[(winPos - i) & oldMask]; - ::MidFree(_window); - } - - _window = win; - _winSizeAllocated = newSize; - _winSize = newSize; + size_t newSize_small = newSize; + const size_t k_Win_AlignSize = 1u << 18; + /* here we add (1 << 7) instead of (COPY_CHUNK_SIZE - 1), because + we want to get same (_winSize) for different COPY_CHUNK_SIZE values. */ + // newSize += (COPY_CHUNK_SIZE - 1) + (k_Win_AlignSize - 1); // for debug : we can get smallest (_winSize) + newSize += (1 << 7) + k_Win_AlignSize; + newSize &= ~(size_t)(k_Win_AlignSize - 1); + if (newSize < newSize_small) + return E_OUTOFMEMORY; } - - _winMask = _winSize - 1; - _winPos &= _winMask; - - if (!_inputBuf) + // (!_isSolid || !_window) + const size_t allocSize = newSize + kMaxMatchLen + 64; + if (allocSize < newSize) + return E_OUTOFMEMORY; + if (!_window || allocSize > _winSize_Allocated) { - _inputBuf = (Byte *)::MidAlloc(kInputBufSize); - if (!_inputBuf) + Z7_RAR_FREE_WINDOW + _window = NULL; + _winSize_Allocated = 0; + Byte *win = (Byte *)::BigAlloc(allocSize); + if (!win) return E_OUTOFMEMORY; + _window = win; + _winSize_Allocated = allocSize; } - - _inStream = inStream; - _outStream = outStream; - - /* - _packSize = 0; - _packSize_Defined = (inSize != NULL); - if (_packSize_Defined) - _packSize = *inSize; - */ - - _unpackSize = 0; - _unpackSize_Defined = (outSize != NULL); - if (_unpackSize_Defined) - _unpackSize = *outSize; - - if ((Int64)_unpackSize >= 0) - _lzEnd += _unpackSize; - else - _lzEnd = 0; - - _progress = progress; - - const HRESULT res = CodeReal(); - - if (res != S_OK) - return res; - if (_lzError) - return S_FALSE; - if (_unsupportedFilter) - return E_NOTIMPL; - return S_OK; + _winSize = newSize; } - // catch(const CInBufferException &e) { return e.ErrorCode; } - // catch(...) { return S_FALSE; } - catch(...) { return E_OUTOFMEMORY; } - // CNewException is possible here. But probably CNewException is caused - // by error in data stream. + + if (!_inputBuf) + { + _inputBuf = (Byte *)z7_AlignedAlloc(kInputBufSize + kInputBufferPadZone); + if (!_inputBuf) + return E_OUTOFMEMORY; + } + + _inStream = inStream; + _outStream = outStream; + _progress = progress; + _progress_Pack = 0; + _progress_Unpack = 0; + + const HRESULT res = CodeReal(); + + if (res != S_OK) + return res; + // _lzError = LZ_ERROR_TYPE_HEADER; // for debug + if (_lzError) + return S_FALSE; + if (_unsupportedFilter) + return E_NOTIMPL; + return S_OK; } + Z7_COM7F_IMF(CDecoder::SetDecoderProperties2(const Byte *data, UInt32 size)) { if (size != 2) + return E_INVALIDARG; + const unsigned pow = data[0]; + const unsigned b1 = data[1]; + const unsigned frac = b1 >> 3; + // unsigned pow = 15 + 8; + // unsigned frac = 1; + if (pow + ((frac + 31) >> 5) > MAX_DICT_LOG - 17) + // if (frac + (pow << 8) >= ((8 * 2 + 7) << 5) + 8 / 8) return E_NOTIMPL; - _dictSizeLog = (Byte)((data[0] & 0xF) + 17); - _isSolid = ((data[1] & 1) != 0); + _dictSize = (size_t)(frac + 32) << (pow + 12); + _isSolid = (b1 & 1) != 0; + _is_v7 = (b1 & 2) != 0; + // printf("\ndict size = %p\n", (void *)(size_t)_dictSize); return S_OK; } diff --git a/CPP/7zip/Compress/Rar5Decoder.h b/CPP/7zip/Compress/Rar5Decoder.h index daf05dd..66c1c3c 100644 --- a/CPP/7zip/Compress/Rar5Decoder.h +++ b/CPP/7zip/Compress/Rar5Decoder.h @@ -5,12 +5,8 @@ #ifndef ZIP7_INC_COMPRESS_RAR5_DECODER_H #define ZIP7_INC_COMPRESS_RAR5_DECODER_H -#include "../../../C/CpuArch.h" - #include "../../Common/MyBuffer2.h" #include "../../Common/MyCom.h" -#include "../../Common/MyException.h" -#include "../../Common/MyVector.h" #include "../ICoder.h" @@ -19,170 +15,7 @@ namespace NCompress { namespace NRar5 { -/* -struct CInBufferException: public CSystemException -{ - CInBufferException(HRESULT errorCode): CSystemException(errorCode) {} -}; -*/ - -class CBitDecoder -{ -public: - const Byte *_buf; - unsigned _bitPos; - bool _wasFinished; - Byte _blockEndBits7; - const Byte *_bufCheck2; - const Byte *_bufCheck; - Byte *_bufLim; - Byte *_bufBase; - - UInt64 _processedSize; - UInt64 _blockEnd; - - ISequentialInStream *_stream; - HRESULT _hres; - - void SetCheck2() - { - _bufCheck2 = _bufCheck; - if (_bufCheck > _buf) - { - UInt64 processed = GetProcessedSize_Round(); - if (_blockEnd < processed) - _bufCheck2 = _buf; - else - { - const UInt64 delta = _blockEnd - processed; - if ((size_t)(_bufCheck - _buf) > delta) - _bufCheck2 = _buf + (size_t)delta; - } - } - } - - bool IsBlockOverRead() const - { - const UInt64 v = GetProcessedSize_Round(); - if (v < _blockEnd) - return false; - if (v > _blockEnd) - return true; - return _bitPos > _blockEndBits7; - } - - /* - CBitDecoder() throw(): - _buf(0), - _bufLim(0), - _bufBase(0), - _stream(0), - _processedSize(0), - _wasFinished(false) - {} - */ - - void Init() throw() - { - _blockEnd = 0; - _blockEndBits7 = 0; - - _bitPos = 0; - _processedSize = 0; - _buf = _bufBase; - _bufLim = _bufBase; - _bufCheck = _buf; - _bufCheck2 = _buf; - _wasFinished = false; - } - - void Prepare2() throw(); - - void Prepare() throw() - { - if (_buf >= _bufCheck) - Prepare2(); - } - - bool ExtraBitsWereRead() const - { - return _buf >= _bufLim && (_buf > _bufLim || _bitPos != 0); - } - - bool InputEofError() const { return ExtraBitsWereRead(); } - - unsigned GetProcessedBits7() const { return _bitPos; } - UInt64 GetProcessedSize_Round() const { return _processedSize + (size_t)(_buf - _bufBase); } - UInt64 GetProcessedSize() const { return _processedSize + (size_t)(_buf - _bufBase) + ((_bitPos + 7) >> 3); } - - void AlignToByte() - { - _buf += (_bitPos + 7) >> 3; - _bitPos = 0; - } - - Byte ReadByteInAligned() - { - return *_buf++; - } - - UInt32 GetValue(unsigned numBits) const - { - UInt32 v = ((UInt32)_buf[0] << 16) | ((UInt32)_buf[1] << 8) | (UInt32)_buf[2]; - v >>= (24 - numBits - _bitPos); - return v & ((1 << numBits) - 1); - } - - void MovePos(unsigned numBits) - { - _bitPos += numBits; - _buf += (_bitPos >> 3); - _bitPos &= 7; - } - - UInt32 ReadBits9(unsigned numBits) - { - const Byte *buf = _buf; - UInt32 v = ((UInt32)buf[0] << 8) | (UInt32)buf[1]; - v &= ((UInt32)0xFFFF >> _bitPos); - numBits += _bitPos; - v >>= (16 - numBits); - _buf = buf + (numBits >> 3); - _bitPos = numBits & 7; - return v; - } - - UInt32 ReadBits9fix(unsigned numBits) - { - const Byte *buf = _buf; - UInt32 v = ((UInt32)buf[0] << 8) | (UInt32)buf[1]; - const UInt32 mask = ((1 << numBits) - 1); - numBits += _bitPos; - v >>= (16 - numBits); - _buf = buf + (numBits >> 3); - _bitPos = numBits & 7; - return v & mask; - } - - UInt32 ReadBits32(unsigned numBits) - { - const UInt32 mask = ((1 << numBits) - 1); - numBits += _bitPos; - const Byte *buf = _buf; - UInt32 v = GetBe32(buf); - if (numBits > 32) - { - v <<= (numBits - 32); - v |= (UInt32)buf[4] >> (40 - numBits); - } - else - v >>= (32 - numBits); - _buf = buf + (numBits >> 3); - _bitPos = numBits & 7; - return v & mask; - } -}; - +class CBitDecoder; struct CFilter { @@ -196,13 +29,20 @@ struct CFilter const unsigned kNumReps = 4; const unsigned kLenTableSize = 11 * 4; const unsigned kMainTableSize = 256 + 1 + 1 + kNumReps + kLenTableSize; -const unsigned kDistTableSize = 64; +const unsigned kExtraDistSymbols_v7 = 16; +const unsigned kDistTableSize_v6 = 64; +const unsigned kDistTableSize_MAX = 64 + kExtraDistSymbols_v7; const unsigned kNumAlignBits = 4; -const unsigned kAlignTableSize = (1 << kNumAlignBits); -const unsigned kLevelTableSize = 20; -const unsigned kTablesSizesSum = kMainTableSize + kDistTableSize + kAlignTableSize + kLenTableSize; +const unsigned kAlignTableSize = 1 << kNumAlignBits; -const unsigned kNumHuffmanBits = 15; +const unsigned kNumHufBits = 15; + +const unsigned k_NumHufTableBits_Main = 10; +const unsigned k_NumHufTableBits_Dist = 7; +const unsigned k_NumHufTableBits_Len = 7; +const unsigned k_NumHufTableBits_Align = 6; + +const unsigned DICT_SIZE_BITS_MAX = 40; Z7_CLASS_IMP_NOQIB_2( CDecoder @@ -215,26 +55,33 @@ Z7_CLASS_IMP_NOQIB_2( // bool _packSize_Defined; bool _unsupportedFilter; - bool _lzError; + Byte _lzError; bool _writeError; bool _isSolid; - bool _solidAllowed; + // bool _solidAllowed; + bool _is_v7; bool _tableWasFilled; bool _wasInit; - Byte _dictSizeLog; - - // CBitDecoder _bitStream; + Byte _exitType; + + // Byte _dictSizeLog; + size_t _dictSize; Byte *_window; size_t _winPos; size_t _winSize; - size_t _winMask; - + size_t _dictSize_forCheck; + size_t _limit; + const Byte *_buf_Res; UInt64 _lzSize; + size_t _reps[kNumReps]; + unsigned _bitPos_Res; + UInt32 _lastLen; - unsigned _numCorrectDistSymbols; + // unsigned _numCorrectDistSymbols; unsigned _numUnusedFilters; + unsigned _numFilters; UInt64 _lzWritten; UInt64 _lzFileStart; @@ -242,53 +89,39 @@ Z7_CLASS_IMP_NOQIB_2( // UInt64 _packSize; UInt64 _lzEnd; UInt64 _writtenFileSize; - size_t _winSizeAllocated; - - UInt32 _reps[kNumReps]; - UInt32 _lastLen; - UInt64 _filterEnd; - CMidBuffer _filterSrc; - CMidBuffer _filterDst; - - CRecordVector _filters; + UInt64 _progress_Pack; + UInt64 _progress_Unpack; + CAlignedBuffer _filterSrc; + CAlignedBuffer _filterDst; + CFilter *_filters; + size_t _winSize_Allocated; ISequentialInStream *_inStream; ISequentialOutStream *_outStream; ICompressProgressInfo *_progress; Byte *_inputBuf; - NHuffman::CDecoder m_MainDecoder; - NHuffman::CDecoder m_DistDecoder; - NHuffman::CDecoder m_AlignDecoder; - NHuffman::CDecoder m_LenDecoder; - NHuffman::CDecoder m_LevelDecoder; - + NHuffman::CDecoder m_MainDecoder; + NHuffman::CDecoder256 m_DistDecoder; + NHuffman::CDecoder256 m_AlignDecoder; + NHuffman::CDecoder256 m_LenDecoder; + Byte m_LenPlusTable[DICT_SIZE_BITS_MAX]; void InitFilters() { _numUnusedFilters = 0; - _filters.Clear(); - } - - void DeleteUnusedFilters() - { - if (_numUnusedFilters != 0) - { - _filters.DeleteFrontal(_numUnusedFilters); - _numUnusedFilters = 0; - } + _numFilters = 0; } - + void DeleteUnusedFilters(); HRESULT WriteData(const Byte *data, size_t size); HRESULT ExecuteFilter(const CFilter &f); HRESULT WriteBuf(); HRESULT AddFilter(CBitDecoder &_bitStream); - HRESULT ReadTables(CBitDecoder &_bitStream); + HRESULT DecodeLZ2(const CBitDecoder &_bitStream) throw(); HRESULT DecodeLZ(); HRESULT CodeReal(); - public: CDecoder(); ~CDecoder(); diff --git a/CPP/7zip/Compress/XpressDecoder.cpp b/CPP/7zip/Compress/XpressDecoder.cpp index 8816a12..47f6061 100644 --- a/CPP/7zip/Compress/XpressDecoder.cpp +++ b/CPP/7zip/Compress/XpressDecoder.cpp @@ -2,47 +2,301 @@ #include "StdAfx.h" -// #include - #include "../../../C/CpuArch.h" +#include "../../../C/RotateDefs.h" #include "HuffmanDecoder.h" #include "XpressDecoder.h" -namespace NCompress { -namespace NXpress { +#ifdef MY_CPU_LE_UNALIGN + #define Z7_XPRESS_DEC_USE_UNALIGNED_COPY +#endif + +#ifdef Z7_XPRESS_DEC_USE_UNALIGNED_COPY + + #define COPY_CHUNK_SIZE 16 + + #define COPY_CHUNK_4_2(dest, src) \ + { \ + ((UInt32 *)(void *)dest)[0] = ((const UInt32 *)(const void *)src)[0]; \ + ((UInt32 *)(void *)dest)[1] = ((const UInt32 *)(const void *)src)[1]; \ + src += 4 * 2; \ + dest += 4 * 2; \ + } + + /* sse2 doesn't help here in GCC and CLANG. + so we disabled sse2 here */ +#if 0 + #if defined(MY_CPU_AMD64) + #define Z7_XPRESS_DEC_USE_SSE2 + #elif defined(MY_CPU_X86) + #if defined(_MSC_VER) && _MSC_VER >= 1300 && defined(_M_IX86_FP) && (_M_IX86_FP >= 2) \ + || defined(__SSE2__) \ + // || 1 == 1 // for debug only + #define Z7_XPRESS_DEC_USE_SSE2 + #endif + #endif +#endif + + #if defined(MY_CPU_ARM64) + #include + #define COPY_OFFSET_MIN 16 + #define COPY_CHUNK1(dest, src) \ + { \ + vst1q_u8((uint8_t *)(void *)dest, \ + vld1q_u8((const uint8_t *)(const void *)src)); \ + src += 16; \ + dest += 16; \ + } + + #define COPY_CHUNK(dest, src) \ + { \ + COPY_CHUNK1(dest, src) \ + if (dest >= dest_lim) break; \ + COPY_CHUNK1(dest, src) \ + } + + #elif defined(Z7_XPRESS_DEC_USE_SSE2) + #include // sse2 + #define COPY_OFFSET_MIN 16 + + #define COPY_CHUNK1(dest, src) \ + { \ + _mm_storeu_si128((__m128i *)(void *)dest, \ + _mm_loadu_si128((const __m128i *)(const void *)src)); \ + src += 16; \ + dest += 16; \ + } + + #define COPY_CHUNK(dest, src) \ + { \ + COPY_CHUNK1(dest, src) \ + if (dest >= dest_lim) break; \ + COPY_CHUNK1(dest, src) \ + } + + #elif defined(MY_CPU_64BIT) + #define COPY_OFFSET_MIN 8 + + #define COPY_CHUNK(dest, src) \ + { \ + ((UInt64 *)(void *)dest)[0] = ((const UInt64 *)(const void *)src)[0]; \ + ((UInt64 *)(void *)dest)[1] = ((const UInt64 *)(const void *)src)[1]; \ + src += 8 * 2; \ + dest += 8 * 2; \ + } + + #else + #define COPY_OFFSET_MIN 4 + + #define COPY_CHUNK(dest, src) \ + { \ + COPY_CHUNK_4_2(dest, src); \ + COPY_CHUNK_4_2(dest, src); \ + } + + #endif +#endif -struct CBitStream + +#ifndef COPY_CHUNK_SIZE + #define COPY_OFFSET_MIN 4 + #define COPY_CHUNK_SIZE 8 + #define COPY_CHUNK_2(dest, src) \ + { \ + const Byte a0 = src[0]; \ + const Byte a1 = src[1]; \ + dest[0] = a0; \ + dest[1] = a1; \ + src += 2; \ + dest += 2; \ + } + #define COPY_CHUNK(dest, src) \ + { \ + COPY_CHUNK_2(dest, src) \ + COPY_CHUNK_2(dest, src) \ + COPY_CHUNK_2(dest, src) \ + COPY_CHUNK_2(dest, src) \ + } +#endif + + +#define COPY_CHUNKS \ +{ \ + Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE \ + do { COPY_CHUNK(dest, src) } \ + while (dest < dest_lim); \ +} + + +static +Z7_FORCE_INLINE +// Z7_ATTRIB_NO_VECTOR +void CopyMatch_1(Byte *dest, const Byte *dest_lim) { - UInt32 Value; - unsigned BitPos; + const unsigned b0 = dest[-1]; + { +#if defined(Z7_XPRESS_DEC_USE_UNALIGNED_COPY) && (COPY_CHUNK_SIZE == 16) + #if defined(MY_CPU_64BIT) + { + const UInt64 v64 = (UInt64)b0 * 0x0101010101010101; + Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE + do + { + ((UInt64 *)(void *)dest)[0] = v64; + ((UInt64 *)(void *)dest)[1] = v64; + dest += 16; + } + while (dest < dest_lim); + } + #else + { + UInt32 v = b0; + v |= v << 8; + v |= v << 16; + do + { + ((UInt32 *)(void *)dest)[0] = v; + ((UInt32 *)(void *)dest)[1] = v; + dest += 8; + ((UInt32 *)(void *)dest)[0] = v; + ((UInt32 *)(void *)dest)[1] = v; + dest += 8; + } + while (dest < dest_lim); + } + #endif +#else + do + { + dest[0] = (Byte)b0; + dest[1] = (Byte)b0; + dest += 2; + dest[0] = (Byte)b0; + dest[1] = (Byte)b0; + dest += 2; + } + while (dest < dest_lim); +#endif + } +} - UInt32 GetValue(unsigned numBits) const - { - return (Value >> (BitPos - numBits)) & ((1 << numBits) - 1); - } - - void MovePos(unsigned numBits) + +// (offset != 1) +static +Z7_FORCE_INLINE +// Z7_ATTRIB_NO_VECTOR +void CopyMatch_Non1(Byte *dest, size_t offset, const Byte *dest_lim) +{ + const Byte *src = dest - offset; { - BitPos -= numBits; + // (COPY_OFFSET_MIN >= 4) + if (offset >= COPY_OFFSET_MIN) + { + COPY_CHUNKS + // return; + } + else +#if (COPY_OFFSET_MIN > 4) + #if COPY_CHUNK_SIZE < 8 + #error Stop_Compiling_Bad_COPY_CHUNK_SIZE + #endif + if (offset >= 4) + { + Z7_PRAGMA_OPT_DISABLE_LOOP_UNROLL_VECTORIZE + do + { + COPY_CHUNK_4_2(dest, src) + #if COPY_CHUNK_SIZE != 16 + if (dest >= dest_lim) break; + #endif + COPY_CHUNK_4_2(dest, src) + } + while (dest < dest_lim); + // return; + } + else +#endif + { + // (offset < 4) + if (offset == 2) + { +#if defined(Z7_XPRESS_DEC_USE_UNALIGNED_COPY) + UInt32 w0 = GetUi16(src); + w0 += w0 << 16; + do + { + SetUi32(dest, w0) + dest += 4; + } + while (dest < dest_lim); +#else + const unsigned b0 = src[0]; + const Byte b1 = src[1]; + do + { + dest[0] = (Byte)b0; + dest[1] = b1; + dest += 2; + } + while (dest < dest_lim); +#endif + } + else // (offset == 3) + { + const unsigned b0 = src[0]; +#if defined(Z7_XPRESS_DEC_USE_UNALIGNED_COPY) + const unsigned w1 = GetUi16(src + 1); + do + { + dest[0] = (Byte)b0; + SetUi16(dest + 1, (UInt16)w1) + dest += 3; + } + while (dest < dest_lim); +#else + const Byte b1 = src[1]; + const Byte b2 = src[2]; + do + { + dest[0] = (Byte)b0; + dest[1] = b1; + dest[2] = b2; + dest += 3; + } + while (dest < dest_lim); +#endif + } + } } -}; +} + + +namespace NCompress { +namespace NXpress { #define BIT_STREAM_NORMALIZE \ - if (bs.BitPos < 16) { \ + if (BitPos > 16) { \ if (in >= lim) return S_FALSE; \ - bs.Value = (bs.Value << 16) | GetUi16(in); \ - in += 2; bs.BitPos += 16; } - + BitPos -= 16; \ + Value |= (UInt32)GetUi16(in) << BitPos; \ + in += 2; } + +#define MOVE_POS(bs, numBits) \ + BitPos += (unsigned)numBits; \ + Value <<= numBits; \ + + static const unsigned kNumHuffBits = 15; +static const unsigned kNumTableBits = 10; static const unsigned kNumLenBits = 4; static const unsigned kLenMask = (1 << kNumLenBits) - 1; static const unsigned kNumPosSlots = 16; static const unsigned kNumSyms = 256 + (kNumPosSlots << kNumLenBits); -HRESULT Decode(const Byte *in, size_t inSize, Byte *out, size_t outSize) +HRESULT Decode_WithExceedWrite(const Byte *in, size_t inSize, Byte *out, size_t outSize) { - NCompress::NHuffman::CDecoder huff; + NCompress::NHuffman::CDecoder huff; if (inSize < kNumSyms / 2 + 4) return S_FALSE; @@ -50,50 +304,57 @@ HRESULT Decode(const Byte *in, size_t inSize, Byte *out, size_t outSize) Byte levels[kNumSyms]; for (unsigned i = 0; i < kNumSyms / 2; i++) { - const Byte b = in[i]; - levels[(size_t)i * 2] = (Byte)(b & 0xF); + const unsigned b = in[i]; + levels[(size_t)i * 2 ] = (Byte)(b & 0xf); levels[(size_t)i * 2 + 1] = (Byte)(b >> 4); } - if (!huff.BuildFull(levels)) + if (!huff.Build(levels, NHuffman::k_BuildMode_Full)) return S_FALSE; } + UInt32 Value; + unsigned BitPos; // how many bits in (Value) were processed - CBitStream bs; - - const Byte *lim = in + inSize - 1; - + const Byte *lim = in + inSize - 1; // points to last byte in += kNumSyms / 2; - bs.Value = ((UInt32)GetUi16(in) << 16) | GetUi16(in + 2); +#ifdef MY_CPU_LE_UNALIGN + Value = GetUi32(in); + Value = rotlFixed(Value, 16); +#else + Value = ((UInt32)GetUi16(in) << 16) | GetUi16(in + 2); +#endif + in += 4; - bs.BitPos = 32; - - size_t pos = 0; + BitPos = 0; + Byte *dest = out; + const Byte *outLim = out + outSize; for (;;) { - // printf("\n%d", pos); - UInt32 sym = huff.DecodeFull(&bs); - // printf(" sym = %d", sym); + unsigned sym; + Z7_HUFF_DECODE_VAL_IN_HIGH32(sym, &huff, kNumHuffBits, kNumTableBits, + Value, Z7_HUFF_DECODE_ERROR_SYM_CHECK_NO, {}, MOVE_POS, {}, bs) + // 0 < BitPos <= 31 BIT_STREAM_NORMALIZE + // 0 < BitPos <= 16 - if (pos >= outSize) - return (sym == 256 && in == lim + 1) ? S_OK : S_FALSE; + if (dest >= outLim) + return (sym == 256 && Value == 0 && in == lim + 1) ? S_OK : S_FALSE; if (sym < 256) - out[pos++] = (Byte)sym; + *dest++ = (Byte)sym; else { - sym -= 256; - UInt32 dist = sym >> kNumLenBits; - UInt32 len = sym & kLenMask; + const unsigned distBits = (unsigned)(Byte)sym >> kNumLenBits; // (sym - 256) >> kNumLenBits; + UInt32 len = (UInt32)(sym & kLenMask); if (len == kLenMask) { if (in > lim) return S_FALSE; + // here we read input bytes in out-of-order related to main input stream (bits in Value): len = *in++; - if (len == 0xFF) + if (len == 0xff) { if (in >= lim) return S_FALSE; @@ -104,26 +365,33 @@ HRESULT Decode(const Byte *in, size_t inSize, Byte *out, size_t outSize) len += kLenMask; } - bs.BitPos -= dist; - dist = (UInt32)1 << dist; - dist += ((bs.Value >> bs.BitPos) & (dist - 1)); - - BIT_STREAM_NORMALIZE - - if (len + 3 > outSize - pos) - return S_FALSE; - if (dist > pos) + len += 3; + if (len > (size_t)(outLim - dest)) return S_FALSE; - Byte *dest = out + pos; - const Byte *src = dest - dist; - pos += len + 3; - len += 1; - *dest++ = *src++; - *dest++ = *src++; - do - *dest++ = *src++; - while (--len); + if (distBits == 0) + { + // d == 1 + if (dest == out) + return S_FALSE; + Byte *destTemp = dest; + dest += len; + CopyMatch_1(destTemp, dest); + } + else + { + unsigned d = (unsigned)(Value >> (32 - distBits)); + MOVE_POS(bs, distBits) + d += 1u << distBits; + // 0 < BitPos <= 31 + BIT_STREAM_NORMALIZE + // 0 < BitPos <= 16 + if (d > (size_t)(dest - out)) + return S_FALSE; + Byte *destTemp = dest; + dest += len; + CopyMatch_Non1(destTemp, d, dest); + } } } } diff --git a/CPP/7zip/Compress/XpressDecoder.h b/CPP/7zip/Compress/XpressDecoder.h index aaa5b25..7d17602 100644 --- a/CPP/7zip/Compress/XpressDecoder.h +++ b/CPP/7zip/Compress/XpressDecoder.h @@ -3,12 +3,15 @@ #ifndef ZIP7_INC_XPRESS_DECODER_H #define ZIP7_INC_XPRESS_DECODER_H -#include "../../Common/MyWindows.h" +#include "../../Common/MyTypes.h" namespace NCompress { namespace NXpress { -HRESULT Decode(const Byte *in, size_t inSize, Byte *out, size_t outSize); +// (out) buffer size must be larger than (outSize) for the following value: +const unsigned kAdditionalOutputBufSize = 32; + +HRESULT Decode_WithExceedWrite(const Byte *in, size_t inSize, Byte *out, size_t outSize); }} diff --git a/CPP/7zip/Compress/XzDecoder.cpp b/CPP/7zip/Compress/XzDecoder.cpp index 420bd71..62a78bf 100644 --- a/CPP/7zip/Compress/XzDecoder.cpp +++ b/CPP/7zip/Compress/XzDecoder.cpp @@ -26,6 +26,7 @@ static HRESULT SResToHRESULT_Code(SRes res) throw() case SZ_OK: return S_OK; case SZ_ERROR_MEM: return E_OUTOFMEMORY; case SZ_ERROR_UNSUPPORTED: return E_NOTIMPL; + default: break; } return S_FALSE; } diff --git a/CPP/7zip/Compress/XzEncoder.cpp b/CPP/7zip/Compress/XzEncoder.cpp index 33f0bde..1bca322 100644 --- a/CPP/7zip/Compress/XzEncoder.cpp +++ b/CPP/7zip/Compress/XzEncoder.cpp @@ -157,7 +157,7 @@ HRESULT CEncoder::SetCoderProp(PROPID propID, const PROPVARIANT &prop) const int filterId = FilterIdFromName(prop.bstrVal); if (filterId < 0 /* || filterId == XZ_ID_LZMA2 */) return E_INVALIDARG; - id32 = (unsigned)filterId; + id32 = (UInt32)(unsigned)filterId; } } diff --git a/CPP/7zip/Compress/ZDecoder.cpp b/CPP/7zip/Compress/ZDecoder.cpp index 07c5fe5..25eb80e 100644 --- a/CPP/7zip/Compress/ZDecoder.cpp +++ b/CPP/7zip/Compress/ZDecoder.cpp @@ -14,7 +14,7 @@ namespace NCompress { namespace NZ { -static const UInt32 kBufferSize = (1 << 20); +static const size_t kBufferSize = 1 << 20; static const Byte kNumBitsMask = 0x1F; static const Byte kBlockModeMask = 0x80; static const unsigned kNumMinBits = 9; @@ -29,14 +29,15 @@ void CDecoder::Free() CDecoder::~CDecoder() { Free(); } -HRESULT CDecoder::CodeReal(ISequentialInStream *inStream, ISequentialOutStream *outStream, - const UInt64 * /* inSize */, const UInt64 * /* outSize */, ICompressProgressInfo *progress) +HRESULT CDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, + ICompressProgressInfo *progress) { + try { + // PackSize = 0; + CInBuffer inBuffer; COutBuffer outBuffer; - PackSize = 0; - if (!inBuffer.Create(kBufferSize)) return E_OUTOFMEMORY; inBuffer.SetStream(inStream); @@ -54,14 +55,14 @@ HRESULT CDecoder::CodeReal(ISequentialInStream *inStream, ISequentialOutStream * if (buf[0] != 0x1F || buf[1] != 0x9D) return S_FALSE; } - Byte prop = buf[2]; + const Byte prop = buf[2]; if ((prop & 0x60) != 0) return S_FALSE; const unsigned maxbits = prop & kNumBitsMask; if (maxbits < kNumMinBits || maxbits > kNumMaxBits) return S_FALSE; - const UInt32 numItems = 1 << maxbits; + const UInt32 numItems = (UInt32)1 << maxbits; // Speed optimization: blockSymbol can contain unused velue. if (maxbits != _numMaxBits || !_parents || !_suffixes || !_stack) @@ -102,7 +103,7 @@ HRESULT CDecoder::CodeReal(ISequentialInStream *inStream, ISequentialOutStream * const unsigned bytePos = bitPos >> 3; UInt32 symbol = buf[bytePos] | ((UInt32)buf[(size_t)bytePos + 1] << 8) | ((UInt32)buf[(size_t)bytePos + 2] << 16); symbol >>= (bitPos & 7); - symbol &= (1 << numBits) - 1; + symbol &= ((UInt32)1 << numBits) - 1; bitPos += numBits; if (bitPos > numBufBits) break; @@ -152,20 +153,17 @@ HRESULT CDecoder::CodeReal(ISequentialInStream *inStream, ISequentialOutStream * else needPrev = false; } - PackSize = inBuffer.GetProcessedSize(); + // PackSize = inBuffer.GetProcessedSize(); const HRESULT res2 = outBuffer.Flush(); return (res == S_OK) ? res2 : res; -} - -Z7_COM7F_IMF(CDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, - const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress)) -{ - try { return CodeReal(inStream, outStream, inSize, outSize, progress); } + + } catch(const CInBufferException &e) { return e.ErrorCode; } catch(const COutBufferException &e) { return e.ErrorCode; } catch(...) { return S_FALSE; } } + bool CheckStream(const Byte *data, size_t size) { if (size < 3) @@ -178,7 +176,7 @@ bool CheckStream(const Byte *data, size_t size) const unsigned maxbits = prop & kNumBitsMask; if (maxbits < kNumMinBits || maxbits > kNumMaxBits) return false; - const UInt32 numItems = 1 << maxbits; + const UInt32 numItems = (UInt32)1 << maxbits; const UInt32 blockSymbol = ((prop & kBlockModeMask) != 0) ? 256 : ((UInt32)1 << kNumMaxBits); unsigned numBits = kNumMinBits; UInt32 head = (blockSymbol == 256) ? 257 : 256; @@ -202,7 +200,7 @@ bool CheckStream(const Byte *data, size_t size) const unsigned bytePos = bitPos >> 3; UInt32 symbol = buf[bytePos] | ((UInt32)buf[bytePos + 1] << 8) | ((UInt32)buf[bytePos + 2] << 16); symbol >>= (bitPos & 7); - symbol &= (1 << numBits) - 1; + symbol &= ((UInt32)1 << numBits) - 1; bitPos += numBits; if (bitPos > numBufBits) { diff --git a/CPP/7zip/Compress/ZDecoder.h b/CPP/7zip/Compress/ZDecoder.h index 390b013..fd67f8f 100644 --- a/CPP/7zip/Compress/ZDecoder.h +++ b/CPP/7zip/Compress/ZDecoder.h @@ -12,10 +12,8 @@ namespace NZ { // Z decoder decodes Z data stream, including 3 bytes of header. -Z7_CLASS_IMP_COM_1( - CDecoder - , ICompressCoder -) +class CDecoder +{ UInt16 *_parents; Byte *_suffixes; Byte *_stack; @@ -25,10 +23,10 @@ Z7_CLASS_IMP_COM_1( CDecoder(): _parents(NULL), _suffixes(NULL), _stack(NULL), /* _prop(0), */ _numMaxBits(0) {} ~CDecoder(); void Free(); - UInt64 PackSize; + // UInt64 PackSize; - HRESULT CodeReal(ISequentialInStream *inStream, ISequentialOutStream *outStream, - const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress); + HRESULT Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, + ICompressProgressInfo *progress); }; /* diff --git a/CPP/7zip/Compress/ZlibDecoder.cpp b/CPP/7zip/Compress/ZlibDecoder.cpp index a066932..dc01894 100644 --- a/CPP/7zip/Compress/ZlibDecoder.cpp +++ b/CPP/7zip/Compress/ZlibDecoder.cpp @@ -17,25 +17,41 @@ namespace NZlib { #define ADLER_MOD 65521 #define ADLER_LOOP_MAX 5550 -UInt32 Adler32_Update(UInt32 adler, const Byte *buf, size_t size); -UInt32 Adler32_Update(UInt32 adler, const Byte *buf, size_t size) +UInt32 Adler32_Update(UInt32 adler, const Byte *data, size_t size); +UInt32 Adler32_Update(UInt32 adler, const Byte *data, size_t size) { - UInt32 a = adler & 0xFFFF; - UInt32 b = (adler >> 16) & 0xFFFF; - while (size > 0) + if (size == 0) + return adler; + UInt32 a = adler & 0xffff; + UInt32 b = adler >> 16; + do { - const unsigned curSize = (size > ADLER_LOOP_MAX) ? ADLER_LOOP_MAX : (unsigned )size; - unsigned i; - for (i = 0; i < curSize; i++) + size_t cur = size; + if (cur > ADLER_LOOP_MAX) + cur = ADLER_LOOP_MAX; + size -= cur; + const Byte *lim = data + cur; + if (cur >= 4) { - a += buf[i]; - b += a; + lim -= 4 - 1; + do + { + a += data[0]; b += a; + a += data[1]; b += a; + a += data[2]; b += a; + a += data[3]; b += a; + data += 4; + } + while (data < lim); + lim += 4 - 1; } - buf += curSize; - size -= curSize; + if (data != lim) { a += *data++; b += a; + if (data != lim) { a += *data++; b += a; + if (data != lim) { a += *data++; b += a; }}} a %= ADLER_MOD; b %= ADLER_MOD; } + while (size); return (b << 16) + a; } @@ -55,38 +71,63 @@ Z7_COM7F_IMF(CDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress)) { DEFLATE_TRY_BEGIN - if (!AdlerStream) - AdlerStream = AdlerSpec = new COutStreamWithAdler; - if (!DeflateDecoder) - { - DeflateDecoderSpec = new NDeflate::NDecoder::CCOMCoder; - DeflateDecoderSpec->ZlibMode = true; - DeflateDecoder = DeflateDecoderSpec; - } + _inputProcessedSize_Additional = 0; + AdlerStream.Create_if_Empty(); + DeflateDecoder.Create_if_Empty(); + DeflateDecoder->Set_NeedFinishInput(true); if (inSize && *inSize < 2) return S_FALSE; - Byte buf[2]; - RINOK(ReadStream_FALSE(inStream, buf, 2)) - if (!IsZlib(buf)) - return S_FALSE; - - AdlerSpec->SetStream(outStream); - AdlerSpec->Init(); - + { + Byte buf[2]; + RINOK(ReadStream_FALSE(inStream, buf, 2)) + if (!IsZlib(buf)) + return S_FALSE; + } + _inputProcessedSize_Additional = 2; + AdlerStream->SetStream(outStream); + AdlerStream->Init(); + // NDeflate::NDecoder::Code() ignores inSize + /* UInt64 inSize2 = 0; if (inSize) inSize2 = *inSize - 2; - - const HRESULT res = DeflateDecoder->Code(inStream, AdlerStream, inSize ? &inSize2 : NULL, outSize, progress); - AdlerSpec->ReleaseStream(); + */ + const HRESULT res = DeflateDecoder.Interface()->Code(inStream, AdlerStream, + /* inSize ? &inSize2 : */ NULL, outSize, progress); + AdlerStream->ReleaseStream(); if (res == S_OK) { - const Byte *p = DeflateDecoderSpec->ZlibFooter; - const UInt32 adler = GetBe32(p); - if (adler != AdlerSpec->GetAdler()) - return S_FALSE; + UInt32 footer32[1]; + UInt32 processedSize; + RINOK(DeflateDecoder->ReadUnusedFromInBuf(footer32, 4, &processedSize)) + if (processedSize != 4) + { + size_t processedSize2 = 4 - processedSize; + RINOK(ReadStream(inStream, (Byte *)(void *)footer32 + processedSize, &processedSize2)) + _inputProcessedSize_Additional += (Int32)processedSize2; + processedSize += (UInt32)processedSize2; + } + + if (processedSize == 4) + { + const UInt32 adler = GetBe32a(footer32); + if (adler != AdlerStream->GetAdler()) + return S_FALSE; // adler error + } + else if (!IsAdlerOptional) + return S_FALSE; // unexpeced end of stream (can't read adler) + else + { + // IsAdlerOptional == true + if (processedSize != 0) + { + // we exclude adler bytes from processed size: + _inputProcessedSize_Additional -= (Int32)processedSize; + return S_FALSE; + } + } } return res; DEFLATE_TRY_END diff --git a/CPP/7zip/Compress/ZlibDecoder.h b/CPP/7zip/Compress/ZlibDecoder.h index 1dc5c67..5488653 100644 --- a/CPP/7zip/Compress/ZlibDecoder.h +++ b/CPP/7zip/Compress/ZlibDecoder.h @@ -14,8 +14,8 @@ Z7_CLASS_IMP_NOQIB_1( COutStreamWithAdler , ISequentialOutStream ) - CMyComPtr _stream; UInt32 _adler; + CMyComPtr _stream; UInt64 _size; public: void SetStream(ISequentialOutStream *stream) { _stream = stream; } @@ -29,13 +29,20 @@ Z7_CLASS_IMP_NOQIB_1( CDecoder , ICompressCoder ) - COutStreamWithAdler *AdlerSpec; - CMyComPtr AdlerStream; - NCompress::NDeflate::NDecoder::CCOMCoder *DeflateDecoderSpec; - CMyComPtr DeflateDecoder; + CMyComPtr2 AdlerStream; + CMyComPtr2 DeflateDecoder; + Int32 _inputProcessedSize_Additional; public: - UInt64 GetInputProcessedSize() const { return DeflateDecoderSpec->GetInputProcessedSize() + 2; } - UInt64 GetOutputProcessedSize() const { return AdlerSpec->GetSize(); } + bool IsAdlerOptional; + + CDecoder(): IsAdlerOptional(false) {} + UInt64 GetInputProcessedSize() const + { + return (UInt64)( + (Int64)DeflateDecoder->GetInputProcessedSize() + + (Int64)_inputProcessedSize_Additional); + } + UInt64 GetOutputProcessedSize() const { return AdlerStream->GetSize(); } }; static bool inline IsZlib(const Byte *p) diff --git a/CPP/7zip/Compress/ZstdDecoder.cpp b/CPP/7zip/Compress/ZstdDecoder.cpp new file mode 100644 index 0000000..c958475 --- /dev/null +++ b/CPP/7zip/Compress/ZstdDecoder.cpp @@ -0,0 +1,437 @@ +// ZstdDecoder.cpp + +#include "StdAfx.h" + +// #include + +#include "../../../C/Alloc.h" + +#include "../Common/CWrappers.h" +#include "../Common/StreamUtils.h" + +#include "ZstdDecoder.h" + +namespace NCompress { +namespace NZstd { + +static const size_t k_Zstd_BlockSizeMax = 1 << 17; +/* + we set _outStepMask as (k_Zstd_BlockSizeMax - 1), because: + - cycSize in zstd decoder for isCyclicMode is aligned for (1 << 17) only. + So some write sizes will be multiple of ((1 << 17) * n). + - Also it can be optimal to flush data after each block decoding. +*/ + +CDecoder::CDecoder(): + _outStepMask(k_Zstd_BlockSizeMax - 1) // must be = (1 << x) - 1 + , _dec(NULL) + , _inProcessed(0) + , _inBufSize(1u << 19) // larger value will reduce the number of memcpy() calls in CZstdDec code + , _inBuf(NULL) + , FinishMode(false) + , DisableHash(False) + // , DisableHash(True) // for debug : fast decoding without hash calculation +{ + // ZstdDecInfo_Clear(&ResInfo); +} + +CDecoder::~CDecoder() +{ + if (_dec) + ZstdDec_Destroy(_dec); + MidFree(_inBuf); +} + + +Z7_COM7F_IMF(CDecoder::SetInBufSize(UInt32 , UInt32 size)) + { _inBufSize = size; return S_OK; } +Z7_COM7F_IMF(CDecoder::SetOutBufSize(UInt32 , UInt32 size)) +{ + // we round it down: + size >>= 1; + size |= size >> (1 << 0); + size |= size >> (1 << 1); + size |= size >> (1 << 2); + size |= size >> (1 << 3); + size |= size >> (1 << 4); + _outStepMask = size; // it's (1 << x) - 1 now + return S_OK; +} + +Z7_COM7F_IMF(CDecoder::SetDecoderProperties2(const Byte * /* prop */, UInt32 /* size */)) +{ + // if (size != 3 && size != 5) return E_NOTIMPL; + return S_OK; +} + + +Z7_COM7F_IMF(CDecoder::SetFinishMode(UInt32 finishMode)) +{ + FinishMode = (finishMode != 0); + // FinishMode = false; // for debug + return S_OK; +} + + +Z7_COM7F_IMF(CDecoder::ReadUnusedFromInBuf(void *data, UInt32 size, UInt32 *processedSize)) +{ + size_t cur = ZstdDec_ReadUnusedFromInBuf(_dec, _afterDecoding_tempPos, data, size); + _afterDecoding_tempPos += cur; + size -= (UInt32)cur; + if (size) + { + const size_t rem = _state.inLim - _state.inPos; + if (size > rem) + size = (UInt32)rem; + if (size) + { + memcpy((Byte *)data + cur, _state.inBuf + _state.inPos, size); + _state.inPos += size; + cur += size; + } + } + *processedSize = (UInt32)cur; + return S_OK; +} + + + +HRESULT CDecoder::Prepare(const UInt64 *outSize) +{ + _inProcessed = 0; + _afterDecoding_tempPos = 0; + ZstdDecState_Clear(&_state); + ZstdDecInfo_CLEAR(&ResInfo) + // _state.outStep = _outStepMask + 1; // must be = (1 << x) + _state.disableHash = DisableHash; + if (outSize) + { + _state.outSize_Defined = True; + _state.outSize = *outSize; + // _state.outSize = 0; // for debug + } + if (!_dec) + { + _dec = ZstdDec_Create(&g_AlignedAlloc, &g_BigAlloc); + if (!_dec) + return E_OUTOFMEMORY; + } + if (!_inBuf || _inBufSize != _inBufSize_Allocated) + { + MidFree(_inBuf); + _inBuf = NULL; + _inBufSize_Allocated = 0; + _inBuf = (Byte *)MidAlloc(_inBufSize); + if (!_inBuf) + return E_OUTOFMEMORY; + _inBufSize_Allocated = _inBufSize; + } + _state.inBuf = _inBuf; + ZstdDec_Init(_dec); + return S_OK; +} + + +Z7_COM7F_IMF(CDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, + const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress)) +{ + RINOK(Prepare(outSize)) + + UInt64 inPrev = 0; + UInt64 outPrev = 0; + UInt64 writtenSize = 0; + bool readWasFinished = false; + SRes sres = SZ_OK; + HRESULT hres = S_OK; + HRESULT hres_Read = S_OK; + + for (;;) + { + if (_state.inPos == _state.inLim && !readWasFinished) + { + _state.inPos = 0; + _state.inLim = _inBufSize; + hres_Read = ReadStream(inStream, _inBuf, &_state.inLim); + // _state.inLim -= 5; readWasFinished = True; // for debug + if (_state.inLim != _inBufSize || hres_Read != S_OK) + { + // hres_Read = 99; // for debug + readWasFinished = True; + } + } + { + const size_t inPos_Start = _state.inPos; + sres = ZstdDec_Decode(_dec, &_state); + _inProcessed += _state.inPos - inPos_Start; + } + /* + if (_state.status == ZSTD_STATUS_FINISHED_FRAME) + printf("\nfinished frame pos=%8x, checksum=%08x\n", (unsigned)_state.outProcessed, (unsigned)_state.info.checksum); + */ + const bool needStop = (sres != SZ_OK) + || _state.status == ZSTD_STATUS_OUT_REACHED + || (outSize && *outSize < _state.outProcessed) + || (readWasFinished && _state.inPos == _state.inLim + && ZstdDecState_DOES_NEED_MORE_INPUT_OR_FINISHED_FRAME(&_state)); + + size_t size = _state.winPos - _state.wrPos; // full write size + if (size) + { + if (!needStop) + { + // we try to flush on aligned positions, if possible + size = _state.needWrite_Size; // minimal required write size + const size_t alignedPos = _state.winPos & ~(size_t)_outStepMask; + if (alignedPos > _state.wrPos) + { + const size_t size2 = alignedPos - _state.wrPos; // optimized aligned size + if (size < size2) + size = size2; + } + } + if (size) + { + { + size_t curSize = size; + if (outSize) + { + const UInt64 rem = *outSize - writtenSize; + if (curSize > rem) + curSize = (size_t)rem; + } + if (curSize) + { + // printf("Write wrPos=%8x, size=%8x\n", (unsigned)_state.wrPos, (unsigned)size); + hres = WriteStream(outStream, _state.win + _state.wrPos, curSize); + if (hres != S_OK) + break; + writtenSize += curSize; // it's real size of data that was written to stream + } + } + _state.wrPos += size; // virtual written size, that will be reported to CZstdDec + // _state.needWrite_Size = 0; // optional + } + } + + if (needStop) + break; + if (progress) + if (_inProcessed - inPrev >= (1 << 27) + || _state.outProcessed - outPrev >= (1 << 28)) + { + inPrev = _inProcessed; + outPrev = _state.outProcessed; + RINOK(progress->SetRatioInfo(&inPrev, &outPrev)) + } + } + + if (hres == S_OK) + { + ZstdDec_GetResInfo(_dec, &_state, sres, &ResInfo); + sres = ResInfo.decode_SRes; + /* now (ResInfo.decode_SRes) can contain 2 extra error codes: + - SZ_ERROR_NO_ARCHIVE : if no frames + - SZ_ERROR_INPUT_EOF : if ZSTD_STATUS_NEEDS_MORE_INPUT + */ + _inProcessed -= ResInfo.extraSize; + if (hres_Read != S_OK && _state.inLim == _state.inPos && readWasFinished) + { + /* if (there is stream reading error, + and decoding was stopped because of end of input stream), + then we use reading error as main error code */ + if (sres == SZ_OK || + sres == SZ_ERROR_INPUT_EOF || + sres == SZ_ERROR_NO_ARCHIVE) + hres = hres_Read; + } + if (sres == SZ_ERROR_INPUT_EOF && !FinishMode) + { + /* SZ_ERROR_INPUT_EOF case is allowed case for (!FinishMode) mode. + So we restore SZ_OK result for that case: */ + ResInfo.decode_SRes = sres = SZ_OK; + } + if (hres == S_OK) + { + hres = SResToHRESULT(sres); + if (hres == S_OK && FinishMode) + { + if ((inSize && *inSize != _inProcessed) + || ResInfo.is_NonFinishedFrame + || (outSize && (*outSize != writtenSize || writtenSize != _state.outProcessed))) + hres = S_FALSE; + } + } + } + return hres; +} + + +Z7_COM7F_IMF(CDecoder::GetInStreamProcessedSize(UInt64 *value)) +{ + *value = _inProcessed; + return S_OK; +} + + +#ifndef Z7_NO_READ_FROM_CODER_ZSTD + +Z7_COM7F_IMF(CDecoder::SetOutStreamSize(const UInt64 *outSize)) +{ + _inProcessed = 0; + _hres_Read = S_OK; + _hres_Decode = S_OK; + _writtenSize = 0; + _readWasFinished = false; + _wasFinished = false; + return Prepare(outSize); +} + + +Z7_COM7F_IMF(CDecoder::SetInStream(ISequentialInStream *inStream)) + { _inStream = inStream; return S_OK; } +Z7_COM7F_IMF(CDecoder::ReleaseInStream()) + { _inStream.Release(); return S_OK; } + + +// if SetInStream() mode: the caller must call GetFinishResult() after full decoding +// to check that there decoding was finished correctly + +HRESULT CDecoder::GetFinishResult() +{ + if (_state.winPos != _state.wrPos || !_wasFinished) + return FinishMode ? S_FALSE : S_OK; + // _state.winPos == _state.wrPos + // _wasFinished == true + if (FinishMode && _hres_Decode == S_OK && _state.outSize_Defined && _state.outSize != _writtenSize) + _hres_Decode = S_FALSE; + return _hres_Decode; +} + + +Z7_COM7F_IMF(CDecoder::Read(void *data, UInt32 size, UInt32 *processedSize)) +{ + if (processedSize) + *processedSize = 0; + + for (;;) + { + if (_state.outSize_Defined) + { + // _writtenSize <= _state.outSize + const UInt64 rem = _state.outSize - _writtenSize; + if (size > rem) + size = (UInt32)rem; + } + { + size_t cur = _state.winPos - _state.wrPos; + if (cur) + { + // _state.winPos != _state.wrPos; + // so there is some decoded data that was not written still + if (size == 0) + { + // if (FinishMode) and we are not allowed to write more, then it's data error + if (FinishMode && _state.outSize_Defined && _state.outSize == _writtenSize) + return S_FALSE; + return S_OK; + } + if (cur > size) + cur = (size_t)size; + // cur != 0 + memcpy(data, _state.win + _state.wrPos, cur); + _state.wrPos += cur; + _writtenSize += cur; + data = (void *)((Byte *)data + cur); + if (processedSize) + *processedSize += (UInt32)cur; + size -= (UInt32)cur; + continue; + } + } + + // _state.winPos == _state.wrPos + if (_wasFinished) + { + if (_hres_Decode == S_OK && FinishMode + && _state.outSize_Defined && _state.outSize != _writtenSize) + _hres_Decode = S_FALSE; + return _hres_Decode; + } + + // _wasFinished == false + if (size == 0 && _state.outSize_Defined && _state.outSize != _state.outProcessed) + { + /* size == 0 : so the caller don't need more data now. + _state.outSize > _state.outProcessed : so more data will be requested + later by caller for full processing. + So we exit without ZstdDec_Decode() call, because we don't want + ZstdDec_Decode() to start new block decoding + */ + return S_OK; + } + // size != 0 || !_state.outSize_Defined || _state.outSize == _state.outProcessed) + + if (_state.inPos == _state.inLim && !_readWasFinished) + { + _state.inPos = 0; + _state.inLim = _inBufSize; + _hres_Read = ReadStream(_inStream, _inBuf, &_state.inLim); + if (_state.inLim != _inBufSize || _hres_Read != S_OK) + { + // _hres_Read = 99; // for debug + _readWasFinished = True; + } + } + + SRes sres; + { + const SizeT inPos_Start = _state.inPos; + sres = ZstdDec_Decode(_dec, &_state); + _inProcessed += _state.inPos - inPos_Start; + } + + const bool inFinished = (_state.inPos == _state.inLim) && _readWasFinished; + + _wasFinished = (sres != SZ_OK) + || _state.status == ZSTD_STATUS_OUT_REACHED + || (_state.outSize_Defined && _state.outSize < _state.outProcessed) + || (inFinished + && ZstdDecState_DOES_NEED_MORE_INPUT_OR_FINISHED_FRAME(&_state)); + + if (!_wasFinished) + continue; + + // _wasFinished == true + /* (_state.winPos != _state.wrPos) is possible here. + So we still can have some data to flush, + but we must all result variables . + */ + HRESULT hres = S_OK; + ZstdDec_GetResInfo(_dec, &_state, sres, &ResInfo); + sres = ResInfo.decode_SRes; + _inProcessed -= ResInfo.extraSize; + if (_hres_Read != S_OK && inFinished) + { + if (sres == SZ_OK || + sres == SZ_ERROR_INPUT_EOF || + sres == SZ_ERROR_NO_ARCHIVE) + hres = _hres_Read; + } + if (sres == SZ_ERROR_INPUT_EOF && !FinishMode) + ResInfo.decode_SRes = sres = SZ_OK; + if (hres == S_OK) + { + hres = SResToHRESULT(sres); + if (hres == S_OK && FinishMode) + if (!inFinished + || ResInfo.is_NonFinishedFrame + || (_state.outSize_Defined && _state.outSize != _state.outProcessed)) + hres = S_FALSE; + } + _hres_Decode = hres; + } +} + +#endif + +}} diff --git a/CPP/7zip/Compress/ZstdDecoder.h b/CPP/7zip/Compress/ZstdDecoder.h new file mode 100644 index 0000000..6e15a96 --- /dev/null +++ b/CPP/7zip/Compress/ZstdDecoder.h @@ -0,0 +1,98 @@ +// ZstdDecoder.h + +#ifndef ZIP7_INC_ZSTD_DECODER_H +#define ZIP7_INC_ZSTD_DECODER_H + +#include "../../../C/ZstdDec.h" + +#include "../../Common/MyCom.h" +#include "../ICoder.h" + +namespace NCompress { +namespace NZstd { + +#ifdef Z7_NO_READ_FROM_CODER +#define Z7_NO_READ_FROM_CODER_ZSTD +#endif + +#ifndef Z7_NO_READ_FROM_CODER_ZSTD +// #define Z7_NO_READ_FROM_CODER_ZSTD +#endif + +class CDecoder Z7_final: + public ICompressCoder, + public ICompressSetDecoderProperties2, + public ICompressSetFinishMode, + public ICompressGetInStreamProcessedSize, + public ICompressReadUnusedFromInBuf, + public ICompressSetBufSize, + #ifndef Z7_NO_READ_FROM_CODER_ZSTD + public ICompressSetInStream, + public ICompressSetOutStreamSize, + public ISequentialInStream, + #endif + public CMyUnknownImp +{ + Z7_COM_QI_BEGIN2(ICompressCoder) + Z7_COM_QI_ENTRY(ICompressSetDecoderProperties2) + Z7_COM_QI_ENTRY(ICompressSetFinishMode) + Z7_COM_QI_ENTRY(ICompressGetInStreamProcessedSize) + Z7_COM_QI_ENTRY(ICompressReadUnusedFromInBuf) + Z7_COM_QI_ENTRY(ICompressSetBufSize) + #ifndef Z7_NO_READ_FROM_CODER_ZSTD + Z7_COM_QI_ENTRY(ICompressSetInStream) + Z7_COM_QI_ENTRY(ICompressSetOutStreamSize) + Z7_COM_QI_ENTRY(ISequentialInStream) + #endif + Z7_COM_QI_END + Z7_COM_ADDREF_RELEASE + + Z7_IFACE_COM7_IMP(ICompressCoder) + Z7_IFACE_COM7_IMP(ICompressSetDecoderProperties2) + Z7_IFACE_COM7_IMP(ICompressSetFinishMode) + Z7_IFACE_COM7_IMP(ICompressGetInStreamProcessedSize) + Z7_IFACE_COM7_IMP(ICompressReadUnusedFromInBuf) + Z7_IFACE_COM7_IMP(ICompressSetBufSize) + #ifndef Z7_NO_READ_FROM_CODER_ZSTD + Z7_IFACE_COM7_IMP(ICompressSetOutStreamSize) + Z7_IFACE_COM7_IMP(ICompressSetInStream) + Z7_IFACE_COM7_IMP(ISequentialInStream) + #endif + + HRESULT Prepare(const UInt64 *outSize); + + UInt32 _outStepMask; + CZstdDecHandle _dec; +public: + UInt64 _inProcessed; + CZstdDecState _state; + +private: + UInt32 _inBufSize; + UInt32 _inBufSize_Allocated; + Byte *_inBuf; + size_t _afterDecoding_tempPos; + + #ifndef Z7_NO_READ_FROM_CODER_ZSTD + CMyComPtr _inStream; + HRESULT _hres_Read; + HRESULT _hres_Decode; + UInt64 _writtenSize; + bool _readWasFinished; + bool _wasFinished; + #endif + +public: + bool FinishMode; + Byte DisableHash; + CZstdDecResInfo ResInfo; + + HRESULT GetFinishResult(); + + CDecoder(); + ~CDecoder(); +}; + +}} + +#endif diff --git a/CPP/7zip/Crc.mak b/CPP/7zip/Crc.mak index 815142d..938fc7e 100644 --- a/CPP/7zip/Crc.mak +++ b/CPP/7zip/Crc.mak @@ -1,6 +1,6 @@ C_OBJS = $(C_OBJS) \ $O\7zCrc.obj -!IF "$(PLATFORM)" == "ia64" || "$(PLATFORM)" == "mips" || "$(PLATFORM)" == "arm" || "$(PLATFORM)" == "arm64" +!IF defined(USE_NO_ASM) || defined(USE_C_CRC) || "$(PLATFORM)" == "ia64" || "$(PLATFORM)" == "mips" || "$(PLATFORM)" == "arm" || "$(PLATFORM)" == "arm64" C_OBJS = $(C_OBJS) \ !ELSE ASM_OBJS = $(ASM_OBJS) \ diff --git a/CPP/7zip/Crc64.mak b/CPP/7zip/Crc64.mak index d58a483..27a50d7 100644 --- a/CPP/7zip/Crc64.mak +++ b/CPP/7zip/Crc64.mak @@ -1,6 +1,6 @@ C_OBJS = $(C_OBJS) \ $O\XzCrc64.obj -!IF "$(PLATFORM)" == "ia64" || "$(PLATFORM)" == "mips" || "$(PLATFORM)" == "arm" || "$(PLATFORM)" == "arm64" +!IF defined(USE_NO_ASM) || defined(USE_C_CRC64) || "$(PLATFORM)" == "ia64" || "$(PLATFORM)" == "mips" || "$(PLATFORM)" == "arm" || "$(PLATFORM)" == "arm64" C_OBJS = $(C_OBJS) \ !ELSE ASM_OBJS = $(ASM_OBJS) \ diff --git a/CPP/7zip/Crypto/7zAes.cpp b/CPP/7zip/Crypto/7zAes.cpp index 6b1c648..b203105 100644 --- a/CPP/7zip/Crypto/7zAes.cpp +++ b/CPP/7zip/Crypto/7zAes.cpp @@ -58,7 +58,8 @@ void CKeyInfo::CalcKey() // MY_ALIGN (16) // CSha256 sha; - CAlignedBuffer sha(sizeof(CSha256) + unrollSize + bufSize * 2); + const size_t shaAllocSize = sizeof(CSha256) + unrollSize + bufSize * 2; + CAlignedBuffer1 sha(shaAllocSize); Byte *buf = sha + sizeof(CSha256); memcpy(buf, Salt, SaltSize); @@ -108,7 +109,7 @@ void CKeyInfo::CalcKey() */ Sha256_Final((CSha256 *)(void *)(Byte *)sha, Key); - memset(sha, 0, sha.Size()); + memset(sha, 0, shaAllocSize); } } @@ -255,19 +256,16 @@ Z7_COM7F_IMF(CDecoder::SetDecoderProperties2(const Byte *data, UInt32 size)) if (size == 0) return S_OK; - Byte b0 = data[0]; - + const unsigned b0 = data[0]; _key.NumCyclesPower = b0 & 0x3F; if ((b0 & 0xC0) == 0) return size == 1 ? S_OK : E_INVALIDARG; - if (size <= 1) return E_INVALIDARG; - Byte b1 = data[1]; - - unsigned saltSize = ((b0 >> 7) & 1) + (b1 >> 4); - unsigned ivSize = ((b0 >> 6) & 1) + (b1 & 0x0F); + const unsigned b1 = data[1]; + const unsigned saltSize = ((b0 >> 7) & 1) + (b1 >> 4); + const unsigned ivSize = ((b0 >> 6) & 1) + (b1 & 0x0F); if (size != 2 + saltSize + ivSize) return E_INVALIDARG; diff --git a/CPP/7zip/Crypto/MyAes.cpp b/CPP/7zip/Crypto/MyAes.cpp index 6abc388..f84bca8 100644 --- a/CPP/7zip/Crypto/MyAes.cpp +++ b/CPP/7zip/Crypto/MyAes.cpp @@ -155,22 +155,34 @@ Z7_COM7F_IMF2(UInt32, CAesCtrCoder::Filter(Byte *data, UInt32 size)) #ifdef MY_CPU_X86_OR_AMD64 #define USE_HW_AES #elif defined(MY_CPU_ARM_OR_ARM64) && defined(MY_CPU_LE) - #if defined(__clang__) - #if (__clang_major__ >= 8) // fix that check + + #if defined(__ARM_FEATURE_AES) \ + || defined(__ARM_FEATURE_CRYPTO) + #define USE_HW_AES + #else + #if defined(MY_CPU_ARM64) \ + || defined(__ARM_ARCH) && (__ARM_ARCH >= 4) \ + || defined(Z7_MSC_VER_ORIGINAL) + #if defined(__ARM_FP) && \ + ( defined(Z7_CLANG_VERSION) && (Z7_CLANG_VERSION >= 30800) \ + || defined(__GNUC__) && (__GNUC__ >= 6) \ + ) \ + || defined(Z7_MSC_VER_ORIGINAL) && (_MSC_VER >= 1910) + #if defined(MY_CPU_ARM64) \ + || !defined(Z7_CLANG_VERSION) \ + || defined(__ARM_NEON) && \ + (Z7_CLANG_VERSION < 170000 || \ + Z7_CLANG_VERSION > 170001) #define USE_HW_AES #endif - #elif defined(__GNUC__) - #if (__GNUC__ >= 6) // fix that check - #define USE_HW_AES #endif - #elif defined(_MSC_VER) - #if _MSC_VER >= 1910 - #define USE_HW_AES #endif #endif #endif #ifdef USE_HW_AES +// #pragma message("=== MyAES.c USE_HW_AES === ") + #define SET_AES_FUNC_2(f2) \ if (algo == 2) if (g_Aes_SupportedFunctions_Flags & k_Aes_SupportedFunctions_HW) \ { f = f2; } diff --git a/CPP/7zip/GuiCommon.rc b/CPP/7zip/GuiCommon.rc index b67409b..95654b6 100644 --- a/CPP/7zip/GuiCommon.rc +++ b/CPP/7zip/GuiCommon.rc @@ -54,31 +54,66 @@ LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US #define MY_MODAL_DIALOG_STYLE STYLE DS_MODALFRAME | DS_CENTER | WS_POPUP | WS_CAPTION | WS_SYSMENU - #define MY_MODAL_RESIZE_DIALOG_STYLE MY_MODAL_DIALOG_STYLE | WS_MINIMIZEBOX | WS_MAXIMIZEBOX | WS_SIZEBOX | WS_THICKFRAME #define MY_PAGE_STYLE STYLE WS_CHILD | WS_DISABLED | WS_CAPTION #define MY_FONT FONT 8, "MS Shell Dlg" +#define MY_MODAL_DIALOG_POSTFIX 0, 0, xs, ys MY_MODAL_DIALOG_STYLE MY_FONT + #define SMALL_PAGE_SIZE_X 120 // #define MY_DIALOG DIALOG 0, 0, xs, ys MY_MODAL_DIALOG_STYLE MY_FONT // #define MY_RESIZE_DIALOG DIALOG 0, 0, xs, ys MY_MODAL_RESIZE_DIALOG_STYLE MY_FONT -#define MY_PAGE DIALOG 0, 0, xs, ys MY_PAGE_STYLE MY_FONT +#define MY_PAGE_POSTFIX 0, 0, xs, ys MY_PAGE_STYLE MY_FONT +#define MY_PAGE DIALOG MY_PAGE_POSTFIX #define OK_CANCEL \ DEFPUSHBUTTON "OK", IDOK, bx2, by, bxs, bys \ PUSHBUTTON "Cancel", IDCANCEL, bx1, by, bxs, bys +#define CONTINUE_CANCEL \ + DEFPUSHBUTTON "Continue",IDCONTINUE, bx2, by, bxs, bys \ + PUSHBUTTON "Cancel", IDCANCEL, bx1, by, bxs, bys + #define MY_BUTTON__CLOSE \ DEFPUSHBUTTON "&Close", IDCLOSE, bx1, by, bxs, bys #define MY_COMBO CBS_DROPDOWNLIST | WS_VSCROLL | WS_TABSTOP #define MY_COMBO_SORTED MY_COMBO | CBS_SORT -#define MY_COMBO_WITH_EDIT CBS_DROPDOWN | CBS_AUTOHSCROLL | WS_VSCROLL | WS_TABSTOP +#define MY_COMBO_WITH_EDIT CBS_DROPDOWN | CBS_AUTOHSCROLL | WS_VSCROLL | WS_TABSTOP #define MY_CHECKBOX "Button", BS_AUTOCHECKBOX | WS_TABSTOP +#define cboxColonSize 18 +#define colonString ":" + +#define MY_CONTROL_CHECKBOX(_text,_id,_x,_y,_xsize) CONTROL _text, _id, MY_CHECKBOX, _x,_y,_xsize,10 +#define MY_CONTROL_CHECKBOX_2LINES(_text,_id,_x,_y,_xsize) CONTROL _text, _id, MY_CHECKBOX | BS_MULTILINE, _x,_y,_xsize,16 +#define MY_CONTROL_CHECKBOX_COLON(_id,_x,_y) MY_CONTROL_CHECKBOX( colonString, _id, _x,_y,cboxColonSize) + +#define MY_AUTORADIOBUTTON "Button", BS_AUTORADIOBUTTON +#define MY_AUTORADIOBUTTON_GROUP MY_AUTORADIOBUTTON | WS_GROUP + +#define MY_CONTROL_AUTORADIOBUTTON(_text,_id,_x,_y,_xsize) CONTROL _text, _id, MY_AUTORADIOBUTTON, _x,_y,_xsize,10 +#define MY_CONTROL_AUTORADIOBUTTON_GROUP(_text,_id,_x,_y,_xsize) CONTROL _text, _id, MY_AUTORADIOBUTTON_GROUP, _x,_y,_xsize,10 + #define MY_TEXT_NOPREFIX 8, SS_NOPREFIX + + +// WS_EX_CLIENTEDGE +#define MY_CONTROL_EDIT_WITH_SPIN(_id_edit, _id_spin, _text, _x, _y, _xSize) \ + EDITTEXT _id_edit, _x, _y, _xSize, 12, ES_CENTER | ES_NUMBER | ES_AUTOHSCROLL \ + CONTROL _text, _id_spin, L"msctls_updown32", \ + UDS_SETBUDDYINT \ + | UDS_ALIGNRIGHT \ + | UDS_AUTOBUDDY \ + | UDS_ARROWKEYS \ + | UDS_NOTHOUSANDS, \ + _x + _xSize, _y, 8, 12 // these values are unused + + +#define OPTIONS_PAGE_XC_SIZE 280 +#define OPTIONS_PAGE_YC_SIZE 280 diff --git a/CPP/7zip/Guid.txt b/CPP/7zip/Guid.txt index eee28f7..abbaf20 100644 --- a/CPP/7zip/Guid.txt +++ b/CPP/7zip/Guid.txt @@ -94,6 +94,8 @@ 04 IArchiveKeepModeForNextOpen 05 IArchiveAllowTail + 09 IArchiveRequestMemoryUseCallback + 10 IArchiveOpenCallback 20 IArchiveExtractCallback @@ -176,7 +178,9 @@ Handler GUIDs: 0B lzma86 0C xz 0D ppmd + 0E zstd + BF LVM C0 AVB C1 LP C2 Sparse diff --git a/CPP/7zip/IDecl.h b/CPP/7zip/IDecl.h index c4d4541..4dbf1eb 100644 --- a/CPP/7zip/IDecl.h +++ b/CPP/7zip/IDecl.h @@ -3,7 +3,7 @@ #ifndef ZIP7_INC_IDECL_H #define ZIP7_INC_IDECL_H -#include "../Common/Common.h" +#include "../Common/Common0.h" #include "../Common/MyUnknown.h" #define k_7zip_GUID_Data1 0x23170F69 diff --git a/CPP/7zip/IStream.h b/CPP/7zip/IStream.h index 4a58bc9..0c44a91 100644 --- a/CPP/7zip/IStream.h +++ b/CPP/7zip/IStream.h @@ -3,6 +3,7 @@ #ifndef ZIP7_INC_ISTREAM_H #define ZIP7_INC_ISTREAM_H +#include "../Common/Common0.h" #include "../Common/MyTypes.h" #include "../Common/MyWindows.h" @@ -145,10 +146,18 @@ IStreamSetRestriction::SetRestriction(UInt64 begin, UInt64 end) It sets region of data in output stream that is restricted. For restricted region it's expected (or allowed) - to change data with different calls of Write()/SetSize(). + that the caller can write to same region with different calls of Write()/SetSize(). Another regions of output stream will be supposed as non-restricted: - The callee usually doesn't flush the data in restricted region. - - The callee usually can flush data from non-restricted region. + - The callee usually can flush data from non-restricted region after writing. + +Actual restiction rules depend also from current stream position. +It's recommended to call SetRestriction() just before the Write() call. +So the callee can optimize writing and flushing, if that Write() +operation is not restricted. + +Note: Each new call of SetRestriction() sets new restictions, +so previous restrction calls has no effect anymore. inputs: @@ -156,35 +165,29 @@ IStreamSetRestriction::SetRestriction(UInt64 begin, UInt64 end) if (begin == end) { - it means that there is no region restriction for flushing. - The callee is allowed to flush already written data and - is allowed to flush all data in future calls of - ISequentialOutStream::Write(), but before next call of SetRestriction(). - The pair of values (begin == 0 && end == 0) is recommended to - remove all restrictions. + No restriction. + The caller will call Write() in sequential order. + After SetRestriction(begin, begin), but before next call of SetRestriction() + { + Additional condition: + it's expected that current stream seek position is equal to stream size. + The callee can make final flushing for any data before current stream seek position. + For each Write(size) call: + The callee can make final flushing for that new written data. + } + The pair of values (begin == 0 && end == 0) is recommended to remove write restriction. } if (begin < end) { - it means that callee must NOT to flush any data in region [begin, end). + it means that callee must NOT flush any data in region [begin, end). The caller is allowed to Seek() to that region and rewrite the data in that restriction region. - if (end == (UInt64(Int64)-1) + if (end == (UInt64)(Int64)-1) { there is no upper bound for restricted region. So non-restricted region will be [0, begin) in that case } - - Note: it's not expected that some already written region was marked as - non-restricted by old call SetRestriction() and later the part of - that region was marked as restricted with new call SetRestriction(). - For example, for different calls with non-empty restricted regions: - (begin_new >= begin_old) is expected : - { - SetRestriction(begin_old, ...) - ... - SetRestriction(begin_new, ...) - } } returns: @@ -195,7 +198,7 @@ IStreamSetRestriction::SetRestriction(UInt64 begin, UInt64 end) Note: IOutStream::SetSize() also can change the data. So it's not expected the call - IOutStream::SetSize() to unrestricted already written region. + IOutStream::SetSize() to region that was written before as unrestricted. */ #define Z7_IFACEM_IStreamSetRestriction(x) \ diff --git a/CPP/7zip/LzmaDec.mak b/CPP/7zip/LzmaDec.mak index 3beb505..624ea9e 100644 --- a/CPP/7zip/LzmaDec.mak +++ b/CPP/7zip/LzmaDec.mak @@ -1,4 +1,4 @@ -!IF "$(PLATFORM)" == "x64" +!IF "$(PLATFORM)" == "x64" || ("$(PLATFORM)" == "arm64" && !defined(NO_ASM_GNU)) !IFNDEF NO_ASM CFLAGS_C_SPEC = -DZ7_LZMA_DEC_OPT ASM_OBJS = $(ASM_OBJS) \ diff --git a/CPP/7zip/UI/Agent/Agent.cpp b/CPP/7zip/UI/Agent/Agent.cpp index 6761b28..eb77f25 100644 --- a/CPP/7zip/UI/Agent/Agent.cpp +++ b/CPP/7zip/UI/Agent/Agent.cpp @@ -1450,7 +1450,7 @@ void CAgentFolder::GetRealIndices(const UInt32 *indices, UInt32 numItems, bool i realIndices.Add(arcIndex); } - HeapSort(&realIndices.Front(), realIndices.Size()); + HeapSort(realIndices.NonConstData(), realIndices.Size()); } Z7_COM7F_IMF(CAgentFolder::Extract(const UInt32 *indices, @@ -1496,8 +1496,8 @@ Z7_COM7F_IMF(CAgentFolder::Extract(const UInt32 *indices, if (path) { pathU = us2fs(path); - if (!pathU.IsEmpty()) - if (!NFile::NName::IsAltStreamPrefixWithColon(pathU)) + if (!pathU.IsEmpty() + && !NFile::NName::IsAltStreamPrefixWithColon(path)) { NFile::NName::NormalizeDirPathPrefix(pathU); NFile::NDir::CreateComplexDir(pathU); @@ -1552,10 +1552,10 @@ Z7_COM7F_IMF(CAgentFolder::Extract(const UInt32 *indices, { CArchiveExtractCallback_Closer ecsCloser(extractCallbackSpec); - HRESULT res = _agentSpec->GetArchive()->Extract(&realIndices.Front(), + HRESULT res = _agentSpec->GetArchive()->Extract(realIndices.ConstData(), realIndices.Size(), testMode, extractCallback); - HRESULT res2 = ecsCloser.Close(); + const HRESULT res2 = ecsCloser.Close(); if (res == S_OK) res = res2; return res; diff --git a/CPP/7zip/UI/Agent/AgentOut.cpp b/CPP/7zip/UI/Agent/AgentOut.cpp index b5a25a3..903cde2 100644 --- a/CPP/7zip/UI/Agent/AgentOut.cpp +++ b/CPP/7zip/UI/Agent/AgentOut.cpp @@ -326,16 +326,15 @@ Z7_COM7F_IMF(CAgent::DoOperation( CUpdateCallbackAgent updateCallbackAgent; updateCallbackAgent.SetCallback(updateCallback100); - CArchiveUpdateCallback *updateCallbackSpec = new CArchiveUpdateCallback; - CMyComPtr updateCallback(updateCallbackSpec ); + CMyComPtr2_Create updateCallback; - updateCallbackSpec->DirItems = &dirItems; - updateCallbackSpec->ArcItems = &arcItems; - updateCallbackSpec->UpdatePairs = &updatePairs2; + updateCallback->DirItems = &dirItems; + updateCallback->ArcItems = &arcItems; + updateCallback->UpdatePairs = &updatePairs2; - SetInArchiveInterfaces(this, updateCallbackSpec); + SetInArchiveInterfaces(this, updateCallback.ClsPtr()); - updateCallbackSpec->Callback = &updateCallbackAgent; + updateCallback->Callback = &updateCallbackAgent; CByteBuffer processedItems; if (processedPaths) @@ -344,7 +343,7 @@ Z7_COM7F_IMF(CAgent::DoOperation( processedItems.Alloc(num); for (unsigned i = 0; i < num; i++) processedItems[i] = 0; - updateCallbackSpec->ProcessedItemsStatuses = processedItems; + updateCallback->ProcessedItemsStatuses = processedItems; } Z7_DECL_CMyComPtr_QI_FROM( @@ -367,7 +366,7 @@ Z7_COM7F_IMF(CAgent::DoOperation( { FOR_VECTOR (i, m_PropValues) propValues[i] = m_PropValues[i]; - RINOK(setProperties->SetProperties(&names.Front(), propValues, names.Size())) + RINOK(setProperties->SetProperties(names.ConstData(), propValues, names.Size())) } catch(...) { @@ -382,15 +381,14 @@ Z7_COM7F_IMF(CAgent::DoOperation( if (sfxModule != NULL) { - CInFileStream *sfxStreamSpec = new CInFileStream; - CMyComPtr sfxStream(sfxStreamSpec); - if (!sfxStreamSpec->Open(us2fs(sfxModule))) + CMyComPtr2_Create sfxStream; + if (!sfxStream->Open(us2fs(sfxModule))) return E_FAIL; // throw "Can't open sfx module"; RINOK(NCompress::CopyStream(sfxStream, outArchiveStream, NULL)) } - HRESULT res = outArchive->UpdateItems(outArchiveStream, updatePairs2.Size(), updateCallback); + const HRESULT res = outArchive->UpdateItems(outArchiveStream, updatePairs2.Size(), updateCallback); if (res == S_OK && processedPaths) { { @@ -443,8 +441,7 @@ Z7_COM7F_IMF(CAgent::DeleteItems(ISequentialOutStream *outArchiveStream, CRecordVector updatePairs; CUpdateCallbackAgent updateCallbackAgent; updateCallbackAgent.SetCallback(updateCallback100); - CArchiveUpdateCallback *updateCallbackSpec = new CArchiveUpdateCallback; - CMyComPtr updateCallback(updateCallbackSpec); + CMyComPtr2_Create updateCallback; CUIntVector realIndices; _agentFolder->GetRealIndices(indices, numItems, @@ -472,11 +469,11 @@ Z7_COM7F_IMF(CAgent::DeleteItems(ISequentialOutStream *outArchiveStream, up2.SetAs_NoChangeArcItem(i); updatePairs.Add(up2); } - updateCallbackSpec->UpdatePairs = &updatePairs; + updateCallback->UpdatePairs = &updatePairs; - SetInArchiveInterfaces(this, updateCallbackSpec); + SetInArchiveInterfaces(this, updateCallback.ClsPtr()); - updateCallbackSpec->Callback = &updateCallbackAgent; + updateCallback->Callback = &updateCallbackAgent; return CommonUpdate(outArchiveStream, updatePairs.Size(), updateCallback); } @@ -489,8 +486,7 @@ HRESULT CAgent::CreateFolder(ISequentialOutStream *outArchiveStream, CDirItems dirItems; CUpdateCallbackAgent updateCallbackAgent; updateCallbackAgent.SetCallback(updateCallback100); - CArchiveUpdateCallback *updateCallbackSpec = new CArchiveUpdateCallback; - CMyComPtr updateCallback(updateCallbackSpec); + CMyComPtr2_Create updateCallback; UInt32 numItemsInArchive; RINOK(GetArchive()->GetNumberOfItems(&numItemsInArchive)) @@ -526,11 +522,11 @@ HRESULT CAgent::CreateFolder(ISequentialOutStream *outArchiveStream, dirItems.Items.Add(di); - updateCallbackSpec->Callback = &updateCallbackAgent; - updateCallbackSpec->DirItems = &dirItems; - updateCallbackSpec->UpdatePairs = &updatePairs; + updateCallback->Callback = &updateCallbackAgent; + updateCallback->DirItems = &dirItems; + updateCallback->UpdatePairs = &updatePairs; - SetInArchiveInterfaces(this, updateCallbackSpec); + SetInArchiveInterfaces(this, updateCallback.ClsPtr()); return CommonUpdate(outArchiveStream, updatePairs.Size(), updateCallback); } @@ -549,8 +545,7 @@ HRESULT CAgent::RenameItem(ISequentialOutStream *outArchiveStream, CRecordVector updatePairs; CUpdateCallbackAgent updateCallbackAgent; updateCallbackAgent.SetCallback(updateCallback100); - CArchiveUpdateCallback *updateCallbackSpec = new CArchiveUpdateCallback; - CMyComPtr updateCallback(updateCallbackSpec); + CMyComPtr2_Create updateCallback; CUIntVector realIndices; _agentFolder->GetRealIndices(indices, numItems, @@ -596,11 +591,11 @@ HRESULT CAgent::RenameItem(ISequentialOutStream *outArchiveStream, updatePairs.Add(up2); } - updateCallbackSpec->Callback = &updateCallbackAgent; - updateCallbackSpec->UpdatePairs = &updatePairs; - updateCallbackSpec->NewNames = &newNames; + updateCallback->Callback = &updateCallbackAgent; + updateCallback->UpdatePairs = &updatePairs; + updateCallback->NewNames = &newNames; - SetInArchiveInterfaces(this, updateCallbackSpec); + SetInArchiveInterfaces(this, updateCallback.ClsPtr()); return CommonUpdate(outArchiveStream, updatePairs.Size(), updateCallback); } @@ -620,8 +615,7 @@ HRESULT CAgent::CommentItem(ISequentialOutStream *outArchiveStream, CRecordVector updatePairs; CUpdateCallbackAgent updateCallbackAgent; updateCallbackAgent.SetCallback(updateCallback100); - CArchiveUpdateCallback *updateCallbackSpec = new CArchiveUpdateCallback; - CMyComPtr updateCallback(updateCallbackSpec); + CMyComPtr2_Create updateCallback; const int mainRealIndex = _agentFolder->GetRealIndex(indices[0]); @@ -642,12 +636,12 @@ HRESULT CAgent::CommentItem(ISequentialOutStream *outArchiveStream, updatePairs.Add(up2); } - updateCallbackSpec->Callback = &updateCallbackAgent; - updateCallbackSpec->UpdatePairs = &updatePairs; - updateCallbackSpec->CommentIndex = mainRealIndex; - updateCallbackSpec->Comment = &newName; + updateCallback->Callback = &updateCallbackAgent; + updateCallback->UpdatePairs = &updatePairs; + updateCallback->CommentIndex = mainRealIndex; + updateCallback->Comment = &newName; - SetInArchiveInterfaces(this, updateCallbackSpec); + SetInArchiveInterfaces(this, updateCallback.ClsPtr()); return CommonUpdate(outArchiveStream, updatePairs.Size(), updateCallback); } @@ -664,8 +658,7 @@ HRESULT CAgent::UpdateOneFile(ISequentialOutStream *outArchiveStream, CDirItems dirItems; CUpdateCallbackAgent updateCallbackAgent; updateCallbackAgent.SetCallback(updateCallback100); - CArchiveUpdateCallback *updateCallbackSpec = new CArchiveUpdateCallback; - CMyComPtr updateCallback(updateCallbackSpec); + CMyComPtr2_Create updateCallback; UInt32 realIndex; { @@ -702,13 +695,13 @@ HRESULT CAgent::UpdateOneFile(ISequentialOutStream *outArchiveStream, } updatePairs.Add(up2); } - updateCallbackSpec->DirItems = &dirItems; - updateCallbackSpec->Callback = &updateCallbackAgent; - updateCallbackSpec->UpdatePairs = &updatePairs; + updateCallback->DirItems = &dirItems; + updateCallback->Callback = &updateCallbackAgent; + updateCallback->UpdatePairs = &updatePairs; - SetInArchiveInterfaces(this, updateCallbackSpec); + SetInArchiveInterfaces(this, updateCallback.ClsPtr()); - updateCallbackSpec->KeepOriginalItemNames = true; + updateCallback->KeepOriginalItemNames = true; return CommonUpdate(outArchiveStream, updatePairs.Size(), updateCallback); } diff --git a/CPP/7zip/UI/Agent/AgentProxy.cpp b/CPP/7zip/UI/Agent/AgentProxy.cpp index df9f315..176f39b 100644 --- a/CPP/7zip/UI/Agent/AgentProxy.cpp +++ b/CPP/7zip/UI/Agent/AgentProxy.cpp @@ -146,7 +146,7 @@ void CProxyArc::AddRealIndices(unsigned dirIndex, CUIntVector &realIndices) cons int CProxyArc::GetRealIndex(unsigned dirIndex, unsigned index) const { const CProxyDir &dir = Dirs[dirIndex]; - unsigned numDirItems = dir.SubDirs.Size(); + const unsigned numDirItems = dir.SubDirs.Size(); if (index < numDirItems) { const CProxyDir &f = Dirs[dir.SubDirs[index]]; @@ -163,14 +163,14 @@ void CProxyArc::GetRealIndices(unsigned dirIndex, const UInt32 *indices, UInt32 realIndices.Clear(); for (UInt32 i = 0; i < numItems; i++) { - UInt32 index = indices[i]; - unsigned numDirItems = dir.SubDirs.Size(); + const UInt32 index = indices[i]; + const unsigned numDirItems = dir.SubDirs.Size(); if (index < numDirItems) AddRealIndices(dir.SubDirs[index], realIndices); else realIndices.Add(dir.SubFiles[index - numDirItems]); } - HeapSort(&realIndices.Front(), realIndices.Size()); + HeapSort(realIndices.NonConstData(), realIndices.Size()); } /////////////////////////////////////////////// @@ -198,9 +198,9 @@ void CProxyArc::CalculateSizes(unsigned dirIndex, IInArchive *archive) for (i = 0; i < dir.SubFiles.Size(); i++) { - UInt32 index = (UInt32)dir.SubFiles[i]; + const UInt32 index = (UInt32)dir.SubFiles[i]; UInt64 size, packSize; - bool sizeDefined = GetSize(archive, index, kpidSize, size); + const bool sizeDefined = GetSize(archive, index, kpidSize, size); dir.Size += size; GetSize(archive, index, kpidPackSize, packSize); dir.PackSize += packSize; @@ -495,7 +495,7 @@ void CProxyArc2::GetRealIndices(unsigned dirIndex, const UInt32 *indices, UInt32 { AddRealIndices_of_ArcItem(dir.Items[indices[i]], includeAltStreams, realIndices); } - HeapSort(&realIndices.Front(), realIndices.Size()); + HeapSort(realIndices.NonConstData(), realIndices.Size()); } void CProxyArc2::CalculateSizes(unsigned dirIndex, IInArchive *archive) @@ -511,7 +511,7 @@ void CProxyArc2::CalculateSizes(unsigned dirIndex, IInArchive *archive) { UInt32 index = dir.Items[i]; UInt64 size, packSize; - bool sizeDefined = GetSize(archive, index, kpidSize, size); + const bool sizeDefined = GetSize(archive, index, kpidSize, size); dir.Size += size; GetSize(archive, index, kpidPackSize, packSize); dir.PackSize += packSize; diff --git a/CPP/7zip/UI/Client7z/Client7z.cpp b/CPP/7zip/UI/Client7z/Client7z.cpp index 121a2f1..b1e43da 100644 --- a/CPP/7zip/UI/Client7z/Client7z.cpp +++ b/CPP/7zip/UI/Client7z/Client7z.cpp @@ -23,6 +23,11 @@ #include "../../Archive/IArchive.h" +#if 0 +// for password request functions: +#include "../../UI/Console/UserInputUtils.h" +#endif + #include "../../IPassword.h" #include "../../../../C/7zVersion.h" @@ -32,6 +37,8 @@ HINSTANCE g_hInstance; HINSTANCE g_hInstance = NULL; #endif +Z7_DIAGNOSTIC_IGNORE_CAST_FUNCTION + // You can find full list of all GUIDs supported by 7-Zip in Guid.txt file. // 7z format GUID: {23170F69-40C1-278A-1000-000110070000} @@ -205,10 +212,13 @@ Z7_COM7F_IMF(CArchiveOpenCallback::CryptoGetTextPassword(BSTR *password)) if (!PasswordIsDefined) { // You can ask real password here from user - // Password = GetPassword(OutStream); - // PasswordIsDefined = true; +#if 0 + RINOK(GetPassword_HRESULT(&g_StdOut, Password)) + PasswordIsDefined = true; +#else PrintError("Password is not defined"); return E_ABORT; +#endif } return StringToBstr(Password, password); } @@ -468,7 +478,7 @@ Z7_COM7F_IMF(CArchiveExtractCallback::GetStream(UInt32 index, _outFileStreamSpec = new COutFileStream; CMyComPtr outStreamLoc(_outFileStreamSpec); - if (!_outFileStreamSpec->Open(fullProcessedPath, CREATE_ALWAYS)) + if (!_outFileStreamSpec->Create_ALWAYS(fullProcessedPath)) { PrintError("Cannot open output file", fullProcessedPath); return E_ABORT; @@ -574,11 +584,14 @@ Z7_COM7F_IMF(CArchiveExtractCallback::CryptoGetTextPassword(BSTR *password)) { if (!PasswordIsDefined) { +#if 0 // You can ask real password here from user - // Password = GetPassword(OutStream); - // PasswordIsDefined = true; + RINOK(GetPassword_HRESULT(&g_StdOut, Password)) + PasswordIsDefined = true; +#else PrintError("Password is not defined"); return E_ABORT; +#endif } return StringToBstr(Password, password); } @@ -773,7 +786,7 @@ Z7_COM7F_IMF(CArchiveUpdateCallback::GetVolumeStream(UInt32 index, ISequentialOu fileName += VolExt; COutFileStream *streamSpec = new COutFileStream; CMyComPtr streamLoc(streamSpec); - if (!streamSpec->Create(us2fs(fileName), false)) + if (!streamSpec->Create_NEW(us2fs(fileName))) return GetLastError_noZero_HRESULT(); *volumeStream = streamLoc.Detach(); return S_OK; @@ -785,11 +798,13 @@ Z7_COM7F_IMF(CArchiveUpdateCallback::CryptoGetTextPassword2(Int32 *passwordIsDef { if (AskPassword) { - // You can ask real password here from user - // Password = GetPassword(OutStream); - // PasswordIsDefined = true; +#if 0 + RINOK(GetPassword_HRESULT(&g_StdOut, Password)) + PasswordIsDefined = true; +#else PrintError("Password is not defined"); return E_ABORT; +#endif } } *passwordIsDefined = BoolToInt(PasswordIsDefined); @@ -839,6 +854,14 @@ int Z7_CDECL main(int numArgs, const char *args[]) return 1; } +#if defined(__clang__) +#pragma GCC diagnostic ignored "-Wc++98-compat-pedantic" +#endif + +#ifdef _WIN32 +Z7_DIAGNOSTIC_IGNORE_CAST_FUNCTION +#endif + Func_CreateObject f_CreateObject = Z7_GET_PROC_ADDRESS( Func_CreateObject, lib.Get_HMODULE(), @@ -929,7 +952,7 @@ int Z7_CDECL main(int numArgs, const char *args[]) COutFileStream *outFileStreamSpec = new COutFileStream; CMyComPtr outFileStream = outFileStreamSpec; - if (!outFileStreamSpec->Create(archiveName, false)) + if (!outFileStreamSpec->Create_NEW(archiveName)) { PrintError("can't create archive file"); return 1; @@ -1058,7 +1081,7 @@ int Z7_CDECL main(int numArgs, const char *args[]) // Get uncompressed size of file NCOM::CPropVariant prop; archive->GetProperty(i, kpidSize, &prop); - char s[32]; + char s[64]; ConvertPropVariantToShortString(prop, s); Print(s); Print(" "); diff --git a/CPP/7zip/UI/Client7z/Client7z.dsp b/CPP/7zip/UI/Client7z/Client7z.dsp index d46300f..1b5826f 100644 --- a/CPP/7zip/UI/Client7z/Client7z.dsp +++ b/CPP/7zip/UI/Client7z/Client7z.dsp @@ -168,6 +168,10 @@ SOURCE=..\..\..\Windows\PropVariantConv.h # End Source File # Begin Source File +SOURCE=..\..\..\Windows\TimeUtils.cpp +# End Source File +# Begin Source File + SOURCE=..\..\..\Windows\TimeUtils.h # End Source File # End Group diff --git a/CPP/7zip/UI/Client7z/makefile b/CPP/7zip/UI/Client7z/makefile index 988701e..9eff262 100644 --- a/CPP/7zip/UI/Client7z/makefile +++ b/CPP/7zip/UI/Client7z/makefile @@ -21,6 +21,7 @@ WIN_OBJS = \ $O\FileName.obj \ $O\PropVariant.obj \ $O\PropVariantConv.obj \ + $O\TimeUtils.obj \ 7ZIP_COMMON_OBJS = \ $O\FileStreams.obj \ diff --git a/CPP/7zip/UI/Common/ArchiveCommandLine.cpp b/CPP/7zip/UI/Common/ArchiveCommandLine.cpp index 2bdbc41..f35433b 100644 --- a/CPP/7zip/UI/Common/ArchiveCommandLine.cpp +++ b/CPP/7zip/UI/Common/ArchiveCommandLine.cpp @@ -27,6 +27,7 @@ #include "../../../Windows/ErrorMsg.h" #include "../../../Windows/FileDir.h" #include "../../../Windows/FileName.h" +#include "../../../Windows/PropVariantConv.h" #include "../../../Windows/System.h" #ifdef _WIN32 #include "../../../Windows/FileMapping.h" @@ -135,6 +136,7 @@ enum Enum kHash, // kHashGenFile, kHashDir, + kExtractMemLimit, kStdIn, kStdOut, @@ -144,6 +146,8 @@ enum Enum kConsoleCharSet, kTechMode, kListFields, + kListPathSlash, + kListTimestampUTC, kPreserveATime, kShareForWrite, @@ -283,6 +287,7 @@ static const CSwitchForm kSwitchForms[] = { "scrc", SWFRM_STRING_MULT(0) }, // { "scrf", SWFRM_STRING_SINGL(1) }, { "shd", SWFRM_STRING_SINGL(1) }, + { "smemx", SWFRM_STRING }, { "si", SWFRM_STRING }, { "so", SWFRM_SIMPLE }, @@ -292,6 +297,8 @@ static const CSwitchForm kSwitchForms[] = { "scc", SWFRM_STRING }, { "slt", SWFRM_SIMPLE }, { "slf", SWFRM_STRING_SINGL(1) }, + { "slsl", SWFRM_MINUS }, + { "slmu", SWFRM_MINUS }, { "ssp", SWFRM_SIMPLE }, { "ssw", SWFRM_SIMPLE }, @@ -457,6 +464,7 @@ static void AddNameToCensor(NWildcard::CCensor &censor, censor.AddPreItem(nop.Include, name, props); } +#ifndef Z7_EXTRACT_ONLY static void AddRenamePair(CObjectVector *renamePairs, const UString &oldName, const UString &newName, NRecursedType::EEnum type, bool wildcardMatching) @@ -481,6 +489,7 @@ static void AddRenamePair(CObjectVector *renamePairs, throw CArcCmdLineException("Unsupported rename command:", val); } } +#endif static void AddToCensorFromListFile( CObjectVector *renamePairs, @@ -507,6 +516,7 @@ static void AddToCensorFromListFile( } if (renamePairs) { + #ifndef Z7_EXTRACT_ONLY if ((names.Size() & 1) != 0) throw CArcCmdLineException(kIncorrectListFile, fileName); for (unsigned i = 0; i < names.Size(); i += 2) @@ -514,6 +524,9 @@ static void AddToCensorFromListFile( // change type !!!! AddRenamePair(renamePairs, names[i], names[i + 1], nop.RecursedType, nop.WildcardMatching); } + #else + throw "not implemented"; + #endif } else FOR_VECTOR (i, names) @@ -562,6 +575,7 @@ static void AddToCensorFromNonSwitchesStrings( AddToCensorFromListFile(renamePairs, censor, nop, s.Ptr(1), codePage); else if (renamePairs) { + #ifndef Z7_EXTRACT_ONLY if (oldIndex == -1) oldIndex = (int)i; else @@ -571,6 +585,9 @@ static void AddToCensorFromNonSwitchesStrings( // AddRenamePair(renamePairs, nonSwitchStrings[oldIndex], s, type); oldIndex = -1; } + #else + throw "not implemented"; + #endif } else AddNameToCensor(censor, nop, s); @@ -605,10 +622,10 @@ static const char *ParseMapWithPaths( const CNameOption &nop) { UString s (s2); - int pos = s.Find(L':'); + const int pos = s.Find(L':'); if (pos < 0) return k_IncorrectMapCommand; - int pos2 = s.Find(L':', (unsigned)(pos + 1)); + const int pos2 = s.Find(L':', (unsigned)(pos + 1)); if (pos2 < 0) return k_IncorrectMapCommand; @@ -625,7 +642,7 @@ static const char *ParseMapWithPaths( CFileMapping map; if (map.Open(FILE_MAP_READ, GetSystemString(s)) != 0) return "Cannot open mapping"; - LPVOID data = map.Map(FILE_MAP_READ, 0, size); + const LPVOID data = map.Map(FILE_MAP_READ, 0, size); if (!data) return "MapViewOfFile error"; CFileUnmapper unmapper(data); @@ -634,10 +651,10 @@ static const char *ParseMapWithPaths( const wchar_t *p = (const wchar_t *)data; if (*p != 0) // data format marker return "Unsupported Map data"; - UInt32 numChars = size / sizeof(wchar_t); + const UInt32 numChars = size / sizeof(wchar_t); for (UInt32 i = 1; i < numChars; i++) { - wchar_t c = p[i]; + const wchar_t c = p[i]; if (c == 0) { // MessageBoxW(0, name, L"7-Zip", 0); @@ -1002,6 +1019,7 @@ void CArcCmdLineParser::Parse1(const UStringVector &commandStrings, options.IsStdErrTerminal = MY_IS_TERMINAL(stderr); options.HelpMode = parser[NKey::kHelp1].ThereIs || parser[NKey::kHelp2].ThereIs || parser[NKey::kHelp3].ThereIs; + options.YesToAll = parser[NKey::kYes].ThereIs; options.StdInMode = parser[NKey::kStdIn].ThereIs; options.StdOutMode = parser[NKey::kStdOut].ThereIs; @@ -1011,6 +1029,13 @@ void CArcCmdLineParser::Parse1(const UStringVector &commandStrings, const UString &s = parser[NKey::kListFields].PostStrings[0]; options.ListFields = GetAnsiString(s); } + if (parser[NKey::kListPathSlash].ThereIs) + { + options.ListPathSeparatorSlash.Val = !parser[NKey::kListPathSlash].WithMinus; + options.ListPathSeparatorSlash.Def = true; + } + if (parser[NKey::kListTimestampUTC].ThereIs) + g_Timestamp_Show_UTC = !parser[NKey::kListTimestampUTC].WithMinus; options.TechMode = parser[NKey::kTechMode].ThereIs; options.ShowTime = parser[NKey::kShowTime].ThereIs; @@ -1090,7 +1115,7 @@ void CArcCmdLineParser::Parse1(const UStringVector &commandStrings, } - #ifndef UNDER_CE +#ifndef UNDER_CE if (parser[NKey::kAffinity].ThereIs) { @@ -1101,7 +1126,9 @@ void CArcCmdLineParser::Parse1(const UStringVector &commandStrings, a.SetFromWStr_if_Ascii(s); Parse1Log += "Set process affinity mask: "; - #ifdef _WIN32 + bool isError = false; + +#ifdef _WIN32 UInt64 v = 0; { @@ -1111,61 +1138,62 @@ void CArcCmdLineParser::Parse1(const UStringVector &commandStrings, a.Empty(); } if (a.IsEmpty()) - throw CArcCmdLineException("Unsupported switch postfix -stm", s); - + isError = true; + else { - #ifndef _WIN64 +#ifndef _WIN64 if (v >= ((UInt64)1 << 32)) throw CArcCmdLineException("unsupported value -stm", s); - #endif + else +#endif { PrintHex(Parse1Log, v); if (!SetProcessAffinityMask(GetCurrentProcess(), (DWORD_PTR)v)) { - DWORD lastError = GetLastError(); + const DWORD lastError = GetLastError(); Parse1Log += " : ERROR : "; Parse1Log += NError::MyFormatMessage(lastError); } } } - #else // _WIN32 +#else // _WIN32 + if (a.Len() != s.Len()) + isError = true; + else { Parse1Log += a; NSystem::CProcessAffinity aff; aff.CpuZero(); - for (unsigned i = 0; i < a.Len(); i++) + unsigned cpu = 0; + unsigned i = a.Len(); + while (i) { - char c = a[i]; - unsigned v; - if (c >= '0' && c <= '9') v = (unsigned)(c - '0'); - else if (c >= 'A' && c <= 'F') v = 10 + (unsigned)(c - 'A'); - else if (c >= 'a' && c <= 'f') v = 10 + (unsigned)(c - 'a'); - else - throw CArcCmdLineException("Unsupported switch postfix -stm", s); - for (unsigned k = 0; k < 4; k++) - { - const unsigned cpu = (a.Len() - 1 - i) * 4 + k; - if (v & ((unsigned)1 << k)) + unsigned v = (Byte)a[--i]; + Z7_PARSE_HEX_DIGIT(v, { isError = true; break; }) + for (unsigned mask = 1; mask != 1u << 4; mask <<= 1, cpu++) + if (v & mask) aff.CpuSet(cpu); - } } - + if (!isError) if (!aff.SetProcAffinity()) { - DWORD lastError = GetLastError(); + const DWORD lastError = GetLastError(); Parse1Log += " : ERROR : "; Parse1Log += NError::MyFormatMessage(lastError); } } - #endif // _WIN32 +#endif // _WIN32 + + if (isError) + throw CArcCmdLineException("Unsupported switch postfix -stm", s); Parse1Log.Add_LF(); } } - #endif +#endif } @@ -1218,6 +1246,40 @@ static void SetBoolPair(NCommandLineParser::CParser &parser, unsigned switchID, bp.Val = !parser[switchID].WithMinus; } + +static bool ParseSizeString(const wchar_t *s, UInt64 &res) +{ + const wchar_t *end; + const UInt64 v = ConvertStringToUInt64(s, &end); + if (s == end) + return false; + const wchar_t c = *end; + + if (c == 0) + { + res = v; + return true; + } + if (end[1] != 0) + return false; + + unsigned numBits; + switch (MyCharLower_Ascii(c)) + { + case 'b': numBits = 0; break; + case 'k': numBits = 10; break; + case 'm': numBits = 20; break; + case 'g': numBits = 30; break; + case 't': numBits = 40; break; + default: return false; + } + const UInt64 val2 = v << numBits; + if ((val2 >> numBits) != v) + return false; + res = val2; + return true; +} + void CArcCmdLineParser::Parse2(CArcCmdLineOptions &options) { const UStringVector &nonSwitchStrings = parser.NonSwitchStrings; @@ -1252,6 +1314,13 @@ void CArcCmdLineParser::Parse2(CArcCmdLineOptions &options) if (parser[NKey::kHashDir].ThereIs) options.ExtractOptions.HashDir = parser[NKey::kHashDir].PostStrings[0]; + if (parser[NKey::kExtractMemLimit].ThereIs) + { + const UString &s = parser[NKey::kExtractMemLimit].PostStrings[0]; + if (!ParseSizeString(s, options.ExtractOptions.NtOptions.MemLimit)) + throw CArcCmdLineException("Unsupported -smemx:", s); + } + if (parser[NKey::kElimDup].ThereIs) { options.ExtractOptions.ElimDup.Def = true; @@ -1300,7 +1369,7 @@ void CArcCmdLineParser::Parse2(CArcCmdLineOptions &options) options.ConsoleCodePage = FindCharset(parser, NKey::kConsoleCharSet, true, -1); - UInt32 codePage = (UInt32)FindCharset(parser, NKey::kListfileCharSet, false, CP_UTF8); + const UInt32 codePage = (UInt32)FindCharset(parser, NKey::kListfileCharSet, false, CP_UTF8); bool thereAreSwitchIncludes = false; @@ -1354,9 +1423,6 @@ void CArcCmdLineParser::Parse2(CArcCmdLineOptions &options) nop, thereAreSwitchIncludes, codePage); - options.YesToAll = parser[NKey::kYes].ThereIs; - - #ifndef Z7_NO_CRYPTO options.PasswordEnabled = parser[NKey::kPassword].ThereIs; if (options.PasswordEnabled) diff --git a/CPP/7zip/UI/Common/ArchiveCommandLine.h b/CPP/7zip/UI/Common/ArchiveCommandLine.h index 9e375b5..acee63c 100644 --- a/CPP/7zip/UI/Common/ArchiveCommandLine.h +++ b/CPP/7zip/UI/Common/ArchiveCommandLine.h @@ -65,6 +65,7 @@ struct CArcCmdLineOptions bool ShowDialog; bool TechMode; bool ShowTime; + CBoolPair ListPathSeparatorSlash; CBoolPair NtSecurity; CBoolPair AltStreams; @@ -145,6 +146,12 @@ struct CArcCmdLineOptions LogLevel(0) { + ListPathSeparatorSlash.Val = +#ifdef _WIN32 + false; +#else + true; +#endif } }; diff --git a/CPP/7zip/UI/Common/ArchiveExtractCallback.cpp b/CPP/7zip/UI/Common/ArchiveExtractCallback.cpp index 4b0cbed..2d32694 100644 --- a/CPP/7zip/UI/Common/ArchiveExtractCallback.cpp +++ b/CPP/7zip/UI/Common/ArchiveExtractCallback.cpp @@ -47,7 +47,9 @@ static const char * const kCantRenameFile = "Cannot rename existing file"; static const char * const kCantDeleteOutputFile = "Cannot delete output file"; static const char * const kCantDeleteOutputDir = "Cannot delete output folder"; static const char * const kCantOpenOutFile = "Cannot open output file"; +#ifndef Z7_SFX static const char * const kCantOpenInFile = "Cannot open input file"; +#endif static const char * const kCantSetFileLen = "Cannot set length for output file"; #ifdef SUPPORT_LINKS static const char * const kCantCreateHardLink = "Cannot create hard link"; @@ -124,7 +126,7 @@ static bool FindExt2(const char *p, const UString &name) break; if (c >= 0x80) return false; - s += (char)MyCharLower_Ascii((char)c); + s.Add_Char((char)MyCharLower_Ascii((char)c)); } for (unsigned i = 0; p[i] != 0;) { @@ -161,10 +163,12 @@ void ReadZoneFile_Of_BaseFile(CFSTR fileName2, CByteBuffer &buf) buf.Free(); } -static bool WriteZoneFile(CFSTR fileName, const CByteBuffer &buf) +bool WriteZoneFile_To_BaseFile(CFSTR fileName, const CByteBuffer &buf) { + FString path (fileName); + path += k_ZoneId_StreamName; NIO::COutFile file; - if (!file.Create(fileName, true)) + if (!file.Create_ALWAYS(path)) return false; return file.WriteFull(buf, buf.Size()); } @@ -275,6 +279,7 @@ CArchiveExtractCallback::CArchiveExtractCallback(): Write_CTime(true), Write_ATime(true), Write_MTime(true), + Is_elimPrefix_Mode(false), _multiArchives(false) { LocalProgressSpec = new CLocalProgress(); @@ -497,16 +502,15 @@ void CArchiveExtractCallback::CreateComplexDirectory(const UStringVector &dirPat } #endif - // bool res = - CreateDir(fullPath); - // if (!res) + HRESULT hres = S_OK; + if (!CreateDir(fullPath)) + hres = GetLastError_noZero_HRESULT(); if (isFinalDir) { if (!NFile::NFind::DoesDirExist(fullPath)) { _itemFailure = true; - SendMessageError("Cannot create folder", fullPath); - // SendMessageError_with_LastError() + SendMessageError_with_Error(hres, "Cannot create folder", fullPath); } } } @@ -544,12 +548,11 @@ HRESULT CArchiveExtractCallback::SendMessageError(const char *message, const FSt return _extractCallback2->MessageError(s); } -HRESULT CArchiveExtractCallback::SendMessageError_with_LastError(const char *message, const FString &path) + +HRESULT CArchiveExtractCallback::SendMessageError_with_Error(HRESULT errorCode, const char *message, const FString &path) { - DWORD errorCode = GetLastError(); - if (errorCode == 0) - errorCode = (DWORD)E_FAIL; UString s (message); + if (errorCode != S_OK) { s += " : "; s += NError::MyFormatMessage(errorCode); @@ -558,6 +561,12 @@ HRESULT CArchiveExtractCallback::SendMessageError_with_LastError(const char *mes return _extractCallback2->MessageError(s); } +HRESULT CArchiveExtractCallback::SendMessageError_with_LastError(const char *message, const FString &path) +{ + const HRESULT errorCode = GetLastError_noZero_HRESULT(); + return SendMessageError_with_Error(errorCode, message, path); +} + HRESULT CArchiveExtractCallback::SendMessageError2(HRESULT errorCode, const char *message, const FString &path1, const FString &path2) { UString s (message); @@ -692,7 +701,7 @@ bool CensorNode_CheckPath2(const NWildcard::CCensorNode &node, const CReadArcIte if (pathParts2.IsEmpty()) pathParts2.AddNew(); UString &back = pathParts2.Back(); - back += ':'; + back.Add_Colon(); back += item.AltStreamName; bool include2; @@ -1064,7 +1073,7 @@ void CArchiveExtractCallback::CorrectPathParts() UString &name = pathParts.Back(); if (needColon) - name += (char)(_ntOptions.ReplaceColonForAltStream ? '_' : ':'); + name.Add_Char((char)(_ntOptions.ReplaceColonForAltStream ? '_' : ':')); name += s; } @@ -1111,6 +1120,7 @@ void CArchiveExtractCallback::CreateFolders() // 21.04 : we don't change original (_item.PathParts) here UStringVector pathParts = _item.PathParts; + // bool is_DirOp = false; if (!pathParts.IsEmpty()) { /* v23: if we extract symlink, and we know that it links to dir: @@ -1125,16 +1135,45 @@ void CArchiveExtractCallback::CreateFolders() #endif ) pathParts.DeleteBack(); + // else is_DirOp = true; } if (pathParts.IsEmpty()) - return; + { + /* if (_some_pathParts_wereRemoved && Is_elimPrefix_Mode), + then we can have empty pathParts() here for root folder. + v24.00: fixed: we set timestamps for such folder still. + */ + if (!_some_pathParts_wereRemoved || + !Is_elimPrefix_Mode) + return; + // return; // ignore empty paths case + } + /* + if (is_DirOp) + { + RINOK(PrepareOperation(NArchive::NExtract::NAskMode::kExtract)) + _op_WasReported = true; + } + */ FString fullPathNew; CreateComplexDirectory(pathParts, fullPathNew); - + + /* + if (is_DirOp) + { + RINOK(SetOperationResult( + // _itemFailure ? NArchive::NExtract::NOperationResult::kDataError : + NArchive::NExtract::NOperationResult::kOK + )) + } + */ + if (!_item.IsDir) return; + if (fullPathNew.IsEmpty()) + return; if (_itemFailure) return; @@ -1326,7 +1365,7 @@ HRESULT CArchiveExtractCallback::GetExtractStream(CMyComPtr outFileStream_Loc(_outFileStreamSpec); - if (!_outFileStreamSpec->Open(fullProcessedPath, _isSplit ? OPEN_ALWAYS: CREATE_ALWAYS)) + if (!_outFileStreamSpec->Create_ALWAYS_or_Open_ALWAYS(fullProcessedPath, !_isSplit)) { // if (::GetLastError() != ERROR_FILE_EXISTS || !isSplit) { @@ -1590,6 +1629,9 @@ Z7_COM7F_IMF(CArchiveExtractCallback::GetStream(UInt32 index, ISequentialOutStre _isSymLinkCreated = false; _itemFailure = false; + _some_pathParts_wereRemoved = false; + // _op_WasReported = false; + #ifdef SUPPORT_LINKS // _copyFile_Path.Empty(); _link.Clear(); @@ -1607,6 +1649,7 @@ Z7_COM7F_IMF(CArchiveExtractCallback::GetStream(UInt32 index, ISequentialOutStre else _extractMode = true; break; + default: break; } @@ -1684,7 +1727,7 @@ Z7_COM7F_IMF(CArchiveExtractCallback::GetStream(UInt32 index, ISequentialOutStre unsigned numRemovePathParts = 0; - switch (_pathMode) + switch ((int)_pathMode) { case NExtract::NPathMode::kFullPaths: case NExtract::NPathMode::kCurPaths: @@ -1730,7 +1773,10 @@ Z7_COM7F_IMF(CArchiveExtractCallback::GetStream(UInt32 index, ISequentialOutStre return E_FAIL; } else + { numRemovePathParts = _removePathParts.Size(); + _some_pathParts_wereRemoved = true; + } break; } @@ -1752,7 +1798,7 @@ Z7_COM7F_IMF(CArchiveExtractCallback::GetStream(UInt32 index, ISequentialOutStre break; } case NExtract::NPathMode::kAbsPaths: - // default: + default: break; } @@ -1777,7 +1823,7 @@ Z7_COM7F_IMF(CArchiveExtractCallback::GetStream(UInt32 index, ISequentialOutStre if (_item.IsAltStream) { if (!pathParts.IsEmpty() || (!_removePartsForAltStreams && _pathMode != NExtract::NPathMode::kNoPathsAlt)) - name += ':'; + name.Add_Colon(); name += _item.AltStreamName; } #endif @@ -1854,6 +1900,7 @@ Z7_COM7F_IMF(CArchiveExtractCallback::PrepareOperation(Int32 askExtractMode)) COM_TRY_BEGIN #ifndef Z7_SFX + // if (!_op_WasReported) if (ExtractToStreamCallback) return ExtractToStreamCallback->PrepareOperation7(askExtractMode); #endif @@ -1868,7 +1915,10 @@ Z7_COM7F_IMF(CArchiveExtractCallback::PrepareOperation(Int32 askExtractMode)) else _extractMode = true; break; + default: break; } + + // if (_op_WasReported) return S_OK; return _extractCallback2->PrepareOperation(_item.Path, BoolToInt(_item.IsDir), askExtractMode, _isSplit ? &_position: NULL); @@ -1912,8 +1962,7 @@ HRESULT CArchiveExtractCallback::CloseFile() FindExt2(kOfficeExtensions, fs2us(_diskFilePath))) { // we must write zone file before setting of timestamps - const FString path = _diskFilePath + k_ZoneId_StreamName; - if (!WriteZoneFile(path, ZoneBuf)) + if (!WriteZoneFile_To_BaseFile(_diskFilePath, ZoneBuf)) { // we can't write it in FAT // SendMessageError_with_LastError("Can't write Zone.Identifier stream", path); @@ -1998,6 +2047,11 @@ HRESULT CArchiveExtractCallback::SetFromLinkPath( const HRESULT errorCode = GetLastError_noZero_HRESULT(); RINOK(SendMessageError2(errorCode, kCantCreateHardLink, fullProcessedPath, existPath)) } + /* + RINOK(PrepareOperation(NArchive::NExtract::NAskMode::kExtract)) + _op_WasReported = true; + RINOK(SetOperationResult(NArchive::NExtract::NOperationResult::kOK)) + */ linkWasSet = true; return S_OK; } @@ -2427,6 +2481,8 @@ Z7_COM7F_IMF(CArchiveExtractCallback::CryptoGetTextPassword(BSTR *password)) } +#ifndef Z7_SFX + // ---------- HASH functions ---------- FString CArchiveExtractCallback::Hash_GetFullFilePath() @@ -2501,6 +2557,79 @@ Z7_COM7F_IMF(CArchiveExtractCallback::ReportOperation( } +Z7_COM7F_IMF(CArchiveExtractCallback::RequestMemoryUse( + UInt32 flags, UInt32 indexType, UInt32 index, const wchar_t *path, + UInt64 requiredSize, UInt64 *allowedSize, UInt32 *answerFlags)) +{ + if ((flags & NRequestMemoryUseFlags::k_IsReport) == 0) + { + const UInt64 memLimit = _ntOptions.MemLimit; + if (memLimit != (UInt64)(Int64)-1) + { + // we overwrite allowedSize + *allowedSize = memLimit; + if (requiredSize <= memLimit) + { + *answerFlags = NRequestMemoryAnswerFlags::k_Allow; + return S_OK; + } + *answerFlags = NRequestMemoryAnswerFlags::k_Limit_Exceeded; + if (flags & NRequestMemoryUseFlags::k_SkipArc_IsExpected) + *answerFlags |= NRequestMemoryAnswerFlags::k_SkipArc; + flags |= NRequestMemoryUseFlags::k_SLimit_Exceeded + | NRequestMemoryUseFlags::k_AllowedSize_WasForced; + } + } + + if (!_requestMemoryUseCallback) + { + _extractCallback2.QueryInterface(IID_IArchiveRequestMemoryUseCallback, + &_requestMemoryUseCallback); + if (!_requestMemoryUseCallback) + { + // keep default (answerFlags) from caller or (answerFlags) that was set in this function + return S_OK; + } + } + +#if 0 + if ((flags & NRequestMemoryUseFlags::k_IsReport) == 0) + if (requiredSize <= *allowedSize) + { + // it's expected, that *answerFlags was set to NRequestMemoryAnswerFlags::k_Allow already, + // because it's default answer for (requiredSize <= *allowedSize) case. + *answerFlags = NRequestMemoryAnswerFlags::k_Allow; // optional code + } + else + { + // we clear *answerFlags, because we want to disable dafault "Allow", if it's set. + // *answerFlags = 0; + /* + NRequestMemoryAnswerFlags::k_SkipArc | + NRequestMemoryAnswerFlags::k_Limit_Exceeded; + */ + } +#endif + + UString s; + if (!path + && indexType == NArchive::NEventIndexType::kInArcIndex + && index != (UInt32)(Int32)-1 + && _arc) + { + RINOK(_arc->GetItem_Path(index, s)) + path = s.Ptr(); + } + + return _requestMemoryUseCallback->RequestMemoryUse( + flags, indexType, index, path, + requiredSize, allowedSize, answerFlags); +} + +#endif // Z7_SFX + + + // ------------ After Extracting functions ------------ void CDirPathSortPair::SetNumSlashes(const FChar *s) diff --git a/CPP/7zip/UI/Common/ArchiveExtractCallback.h b/CPP/7zip/UI/Common/ArchiveExtractCallback.h index 5ed20f3..7eb2f67 100644 --- a/CPP/7zip/UI/Common/ArchiveExtractCallback.h +++ b/CPP/7zip/UI/Common/ArchiveExtractCallback.h @@ -27,9 +27,9 @@ Z7_CLASS_IMP_NOQIB_1( COutStreamWithHash , ISequentialOutStream ) + bool _calculate; CMyComPtr _stream; UInt64 _size; - bool _calculate; public: IHashCalc *_hash; @@ -66,12 +66,15 @@ struct CExtractNtOptions bool PreserveATime; bool OpenShareForWrite; + UInt64 MemLimit; + CExtractNtOptions(): ReplaceColonForAltStream(false), WriteToAltStreamIfColon(false), ExtractOwner(false), PreserveATime(false), - OpenShareForWrite(false) + OpenShareForWrite(false), + MemLimit((UInt64)(Int64)-1) { SymLinks.Val = true; SymLinks_AllowDangerous.Val = false; @@ -249,25 +252,35 @@ class CArchiveExtractCallback Z7_final: public IArchiveExtractCallbackMessage2, public ICryptoGetTextPassword, public ICompressProgressInfo, +#ifndef Z7_SFX public IArchiveUpdateCallbackFile, public IArchiveGetDiskProperty, + public IArchiveRequestMemoryUseCallback, +#endif public CMyUnknownImp { - Z7_COM_UNKNOWN_IMP_5( - /* IArchiveExtractCallback, */ - IArchiveExtractCallbackMessage2, - ICryptoGetTextPassword, - ICompressProgressInfo, - IArchiveUpdateCallbackFile, - IArchiveGetDiskProperty) + /* IArchiveExtractCallback, */ + Z7_COM_QI_BEGIN2(IArchiveExtractCallbackMessage2) + Z7_COM_QI_ENTRY(ICryptoGetTextPassword) + Z7_COM_QI_ENTRY(ICompressProgressInfo) +#ifndef Z7_SFX + Z7_COM_QI_ENTRY(IArchiveUpdateCallbackFile) + Z7_COM_QI_ENTRY(IArchiveGetDiskProperty) + Z7_COM_QI_ENTRY(IArchiveRequestMemoryUseCallback) +#endif + Z7_COM_QI_END + Z7_COM_ADDREF_RELEASE Z7_IFACE_COM7_IMP(IProgress) Z7_IFACE_COM7_IMP(IArchiveExtractCallback) Z7_IFACE_COM7_IMP(IArchiveExtractCallbackMessage2) Z7_IFACE_COM7_IMP(ICryptoGetTextPassword) Z7_IFACE_COM7_IMP(ICompressProgressInfo) +#ifndef Z7_SFX Z7_IFACE_COM7_IMP(IArchiveUpdateCallbackFile) Z7_IFACE_COM7_IMP(IArchiveGetDiskProperty) + Z7_IFACE_COM7_IMP(IArchiveRequestMemoryUseCallback) +#endif const CArc *_arc; CExtractNtOptions _ntOptions; @@ -290,6 +303,11 @@ class CArchiveExtractCallback Z7_final: bool _isSymLinkCreated; bool _itemFailure; + bool _some_pathParts_wereRemoved; +public: + bool Is_elimPrefix_Mode; + +private: bool _curSize_Defined; bool _fileLength_WasSet; @@ -317,6 +335,7 @@ class CArchiveExtractCallback Z7_final: CMyComPtr ExtractToStreamCallback; CGetProp *GetProp_Spec; CMyComPtr GetProp; + CMyComPtr _requestMemoryUseCallback; #endif @@ -418,6 +437,7 @@ class CArchiveExtractCallback Z7_final: public: HRESULT SendMessageError(const char *message, const FString &path); + HRESULT SendMessageError_with_Error(HRESULT errorCode, const char *message, const FString &path); HRESULT SendMessageError_with_LastError(const char *message, const FString &path); HRESULT SendMessageError2(HRESULT errorCode, const char *message, const FString &path1, const FString &path2); @@ -580,5 +600,6 @@ struct CArchiveExtractCallback_Closer bool CensorNode_CheckPath(const NWildcard::CCensorNode &node, const CReadArcItem &item); void ReadZoneFile_Of_BaseFile(CFSTR fileName2, CByteBuffer &buf); +bool WriteZoneFile_To_BaseFile(CFSTR fileName, const CByteBuffer &buf); #endif diff --git a/CPP/7zip/UI/Common/ArchiveName.cpp b/CPP/7zip/UI/Common/ArchiveName.cpp index 1c0c3a8..3c0976d 100644 --- a/CPP/7zip/UI/Common/ArchiveName.cpp +++ b/CPP/7zip/UI/Common/ArchiveName.cpp @@ -158,7 +158,7 @@ UString CreateArchiveName( baseName = name; if (!simple_IsAllowed) { - HeapSort(&ids.Front(), ids.Size()); + HeapSort(ids.NonConstData(), ids.Size()); UInt32 v = 2; const unsigned num = ids.Size(); for (unsigned i = 0; i < num; i++) @@ -169,7 +169,7 @@ UString CreateArchiveName( if (id == v) v = id + 1; } - name += '_'; + name.Add_Char('_'); name.Add_UInt32(v); } return name; diff --git a/CPP/7zip/UI/Common/ArchiveOpenCallback.cpp b/CPP/7zip/UI/Common/ArchiveOpenCallback.cpp index 4e5f1f5..8a72936 100644 --- a/CPP/7zip/UI/Common/ArchiveOpenCallback.cpp +++ b/CPP/7zip/UI/Common/ArchiveOpenCallback.cpp @@ -85,6 +85,7 @@ Z7_COM7F_IMF(COpenCallbackImp::GetProperty(PROPID propID, PROPVARIANT *value)) { case kpidName: prop = _subArchiveName; break; // case kpidSize: prop = _subArchiveSize; break; // we don't use it now + default: break; } else switch (propID) @@ -97,6 +98,7 @@ Z7_COM7F_IMF(COpenCallbackImp::GetProperty(PROPID propID, PROPVARIANT *value)) case kpidCTime: PropVariant_SetFrom_FiTime(prop, _fileInfo.CTime); break; case kpidATime: PropVariant_SetFrom_FiTime(prop, _fileInfo.ATime); break; case kpidMTime: PropVariant_SetFrom_FiTime(prop, _fileInfo.MTime); break; + default: break; } prop.Detach(value); return S_OK; @@ -106,12 +108,15 @@ Z7_COM7F_IMF(COpenCallbackImp::GetProperty(PROPID propID, PROPVARIANT *value)) // ---------- CInFileStreamVol ---------- -Z7_CLASS_IMP_COM_2( - CInFileStreamVol - , IInStream - , IStreamGetSize -) - Z7_IFACE_COM7_IMP(ISequentialInStream) +Z7_class_final(CInFileStreamVol): + public IInStream + , public IStreamGetSize + , public CMyUnknownImp +{ + Z7_IFACES_IMP_UNK_3( + IInStream, + ISequentialInStream, + IStreamGetSize) public: unsigned FileIndex; COpenCallbackImp *OpenCallbackImp; diff --git a/CPP/7zip/UI/Common/ArchiveOpenCallback.h b/CPP/7zip/UI/Common/ArchiveOpenCallback.h index 4e44c9d..a7b8b76 100644 --- a/CPP/7zip/UI/Common/ArchiveOpenCallback.h +++ b/CPP/7zip/UI/Common/ArchiveOpenCallback.h @@ -167,6 +167,7 @@ class COpenCallbackImp Z7_final: COpenCallbackImp(): _subArchiveMode(false), + PasswordWasAsked(false), ReOpenCallback(NULL), Callback(NULL) {} diff --git a/CPP/7zip/UI/Common/Bench.cpp b/CPP/7zip/UI/Common/Bench.cpp index 5da9783..87a2df4 100644 --- a/CPP/7zip/UI/Common/Bench.cpp +++ b/CPP/7zip/UI/Common/Bench.cpp @@ -2,12 +2,9 @@ #include "StdAfx.h" -#include "../../../../C/CpuArch.h" - // #include #ifndef _WIN32 - #define USE_POSIX_TIME #define USE_POSIX_TIME2 #endif // _WIN32 @@ -31,10 +28,12 @@ #else #include #endif +#define BENCH_ALLOCA_VALUE(index) (((index) * 64 * 21) & 0x7FF) #endif #include "../../../../C/7zCrc.h" #include "../../../../C/RotateDefs.h" +#include "../../../../C/CpuArch.h" #ifndef Z7_ST #include "../../../Windows/Synchronization.h" @@ -105,9 +104,9 @@ static const unsigned kOldLzmaDictBits = 32; // static const size_t kAdditionalSize = (size_t)1 << 32; // for debug static const size_t kAdditionalSize = (size_t)1 << 16; -static const UInt32 kCompressedAdditionalSize = (1 << 10); +static const size_t kCompressedAdditionalSize = 1 << 10; -static const UInt32 kMaxMethodPropSize = (1 << 6); +static const UInt32 kMaxMethodPropSize = 1 << 6; #define ALLOC_WITH_HRESULT(_buffer_, _size_) \ @@ -126,32 +125,40 @@ class CBaseRandomGenerator Z7_FORCE_INLINE UInt32 GetRnd() { +#if 0 + // for debug: + return 0x0c080400; + // return 0; +#else return Salt ^ ( ((A1 = 36969 * (A1 & 0xffff) + (A1 >> 16)) << 16) + ((A2 = 18000 * (A2 & 0xffff) + (A2 >> 16)) ) ); +#endif } }; +static const size_t k_RandBuf_AlignMask = 4 - 1; + Z7_NO_INLINE -static void RandGen(Byte *buf, size_t size) +static void RandGen_BufAfterPad(Byte *buf, size_t size) { CBaseRandomGenerator RG; - const size_t size4 = size & ~((size_t)3); - size_t i; - for (i = 0; i < size4; i += 4) + for (size_t i = 0; i < size; i += 4) { const UInt32 v = RG.GetRnd(); - SetUi32(buf + i, v) + SetUi32a(buf + i, v) } + /* UInt32 v = RG.GetRnd(); for (; i < size; i++) { buf[i] = (Byte)v; v >>= 8; } + */ } @@ -159,14 +166,14 @@ class CBenchRandomGenerator: public CMidAlignedBuffer { static UInt32 GetVal(UInt32 &res, unsigned numBits) { - UInt32 val = res & (((UInt32)1 << numBits) - 1); + const UInt32 val = res & (((UInt32)1 << numBits) - 1); res >>= numBits; return val; } static UInt32 GetLen(UInt32 &r) { - UInt32 len = GetVal(r, 2); + const unsigned len = (unsigned)GetVal(r, 2); return GetVal(r, 1 + len); } @@ -282,7 +289,7 @@ Z7_COM7F_IMF(CBenchmarkInStream::Read(void *data, UInt32 size, UInt32 *processed if (size > remain) size = (UInt32)remain; - if (size != 0) + if (size) memcpy(data, Data + Pos, size); Pos += size; @@ -1141,6 +1148,7 @@ class CEncoderInfo Z7_final #ifdef USE_ALLOCA alloca(decoder->AllocaSize); + // printf("\nalloca=%d\n", (unsigned)decoder->AllocaSize); #endif CEncoderInfo *encoder = decoder->Encoder; @@ -1462,14 +1470,22 @@ HRESULT CEncoderInfo::Encode() Byte *filterData = rgCopy; if (i == numIterations - 1 || calcCrc || useRealData_Enc) { + // printf("\nfilterData = (Byte *)*outStreamSpec;\n"); filterData = (Byte *)*outStreamSpec; if (data_Was_Changed) + { + // printf("\nmemcpy(filterData, uncompressedDataPtr\n"); memcpy(filterData, uncompressedDataPtr, kBufferSize); + } data_Was_Changed = true; } _encoderFilter->Init(); if (calcCrc) + { + // printf("\nInitCrc\n"); outStreamSpec->InitCrc(); + } + // printf("\nMy_FilterBench\n"); My_FilterBench(_encoderFilter, filterData, kBufferSize, calcCrc ? &outStreamSpec->Crc : NULL); } @@ -1541,6 +1557,8 @@ HRESULT CEncoderInfo::Decode(UInt32 decoderIndex) else coder = decoder; + // printf("\ndecoderIndex = %d, stack = %p", decoderIndex, &coder); + CMyComPtr setDecProps; coder.QueryInterface(IID_ICompressSetDecoderProperties2, &setDecProps); if (!setDecProps && propStreamSpec->GetPos() != 0) @@ -1927,10 +1945,24 @@ static HRESULT MethodBench( CEncoderInfo &encoder = encoders[i]; encoder.NumIterations = GetNumIterations(benchProps->GetNumCommands_Enc(uncompressedDataSize), complexInCommands); // encoder.NumIterations = 3; - encoder.Salt = g_CrcTable[i & 0xFF]; - encoder.Salt ^= (g_CrcTable[(i >> 8) & 0xFF] << 3); + { +#if 0 + #define kCrcPoly 0xEDB88320 + UInt32 r = i; + unsigned num = numEncoderThreads < 256 ? 8 : 16; + do + r = (r >> 1) ^ (kCrcPoly & ((UInt32)0 - (r & 1))); + while (--num); + encoder.Salt = r; +#else + UInt32 salt0 = g_CrcTable[(Byte)i]; + UInt32 salt1 = g_CrcTable[(Byte)(i >> 8)]; + encoder.Salt = salt0 ^ (salt1 << 3); +#endif + } + // (g_CrcTable[0] == 0), and (encoder.Salt == 0) for first thread - // printf(" %8x", encoder.Salt); + // printf("\n encoder index = %d, Salt = %8x\n", i, encoder.Salt); encoder.KeySize = benchProps->KeySize; @@ -1953,7 +1985,7 @@ static HRESULT MethodBench( if (mtEncMode) { #ifdef USE_ALLOCA - encoder.AllocaSize = (i * 16 * 21) & 0x7FF; + encoder.AllocaSize = BENCH_ALLOCA_VALUE(i); #endif encoder.Common = &encoderFlusher.Common; @@ -2055,7 +2087,7 @@ static HRESULT MethodBench( #ifndef Z7_ST { - int numSubThreads = method.Get_NumThreads(); + const int numSubThreads = method.Get_NumThreads(); encoder.NumDecoderSubThreads = (numSubThreads <= 0) ? 1 : (unsigned)numSubThreads; } if (mtDecoderMode) @@ -2064,7 +2096,7 @@ static HRESULT MethodBench( { const HRESULT res = encoder.CreateDecoderThread(j, (i == 0 && j == 0) #ifdef USE_ALLOCA - , ((i * numSubDecoderThreads + j) * 16 * 21) & 0x7FF + , BENCH_ALLOCA_VALUE(i * numSubDecoderThreads + j) #endif ); RINOK(res) @@ -2200,7 +2232,7 @@ UInt64 GetBenchMemoryUsage(UInt32 numThreads, int level, UInt64 dictionary, bool const int btMode = (algo == 0 ? 0 : 1); UInt32 numBigThreads = numThreads; - bool lzmaMt = (totalBench || (numThreads > 1 && btMode)); + const bool lzmaMt = (totalBench || (numThreads > 1 && btMode)); if (btMode) { if (!totalBench && lzmaMt) @@ -2236,19 +2268,32 @@ struct CCrcInfo_Base }; +// for debug: define it to test hash calling with unaligned data +// #define Z7_BENCH_HASH_ALIGN_BUF_OFFSET 3 + HRESULT CCrcInfo_Base::Generate(const Byte *data, size_t size) { Size = size; Data = data; if (!data || CreateLocalBuf) { - ALLOC_WITH_HRESULT(&Buffer, size) - Data = Buffer; + Byte *buf; + const size_t size2 = (size + k_RandBuf_AlignMask) & ~(size_t)k_RandBuf_AlignMask; + if (size2 < size) + return E_OUTOFMEMORY; +#ifdef Z7_BENCH_HASH_ALIGN_BUF_OFFSET + ALLOC_WITH_HRESULT(&Buffer, size2 + Z7_BENCH_HASH_ALIGN_BUF_OFFSET) + buf = Buffer + Z7_BENCH_HASH_ALIGN_BUF_OFFSET; +#else + ALLOC_WITH_HRESULT(&Buffer, size2) + buf = Buffer; +#endif + Data = buf; + if (!data) + RandGen_BufAfterPad(buf, size); + else if (size != 0) // (CreateLocalBuf == true) + memcpy(buf, data, size); } - if (!data) - RandGen(Buffer, size); - else if (CreateLocalBuf && size != 0) - memcpy(Buffer, data, size); return S_OK; } @@ -2258,13 +2303,13 @@ HRESULT CCrcInfo_Base::CrcProcess(UInt64 numIterations, IBenchPrintCallback *callback) { MY_ALIGN(16) - Byte hash[64]; - memset(hash, 0, sizeof(hash)); + UInt32 hash32[64 / 4]; + memset(hash32, 0, sizeof(hash32)); CheckSum_Res = 0; const UInt32 hashSize = hf->GetDigestSize(); - if (hashSize > sizeof(hash)) + if (hashSize > sizeof(hash32)) return S_FALSE; const Byte *buf = Data; @@ -2274,7 +2319,7 @@ HRESULT CCrcInfo_Base::CrcProcess(UInt64 numIterations, UInt64 prev = 0; UInt64 cur = 0; - for (UInt64 i = 0; i < numIterations; i++) + do { hf->Init(); size_t pos = 0; @@ -2288,12 +2333,12 @@ HRESULT CCrcInfo_Base::CrcProcess(UInt64 numIterations, } while (pos != size); - hf->Final(hash); + hf->Final((Byte *)(void *)hash32); UInt32 sum = 0; for (UInt32 j = 0; j < hashSize; j += 4) { sum = rotlFixed(sum, 11); - sum += GetUi32(hash + j); + sum += GetUi32((const Byte *)(const void *)hash32 + j); } if (checkSum) { @@ -2315,6 +2360,8 @@ HRESULT CCrcInfo_Base::CrcProcess(UInt64 numIterations, } } } + while (--numIterations); + CheckSum_Res = checkSum_Prev; return S_OK; } @@ -2555,6 +2602,7 @@ WRes CCrcThreads::StartAndWait(bool exitMode) #endif +/* static UInt32 CrcCalc1(const Byte *buf, size_t size) { UInt32 crc = CRC_INIT_VAL; @@ -2562,6 +2610,7 @@ static UInt32 CrcCalc1(const Byte *buf, size_t size) crc = CRC_UPDATE_BYTE(crc, buf[i]); return CRC_GET_DIGEST(crc); } +*/ /* static UInt32 RandGenCrc(Byte *buf, size_t size, CBaseRandomGenerator &RG) @@ -2574,25 +2623,21 @@ static UInt32 RandGenCrc(Byte *buf, size_t size, CBaseRandomGenerator &RG) static bool CrcInternalTest() { CAlignedBuffer buffer; - const size_t kBufferSize0 = (1 << 8); - const size_t kBufferSize1 = (1 << 10); - const unsigned kCheckSize = (1 << 5); - buffer.Alloc(kBufferSize0 + kBufferSize1); + const size_t kBufSize = 1 << 11; + const size_t kCheckSize = 1 << 6; + buffer.Alloc(kBufSize); if (!buffer.IsAllocated()) return false; Byte *buf = (Byte *)buffer; - size_t i; - for (i = 0; i < kBufferSize0; i++) - buf[i] = (Byte)i; - UInt32 crc1 = CrcCalc1(buf, kBufferSize0); - if (crc1 != 0x29058C73) - return false; - RandGen(buf + kBufferSize0, kBufferSize1); - for (i = 0; i < kBufferSize0 + kBufferSize1 - kCheckSize; i++) - for (unsigned j = 0; j < kCheckSize; j++) - if (CrcCalc1(buf + i, j) != CrcCalc(buf + i, j)) - return false; - return true; + RandGen_BufAfterPad(buf, kBufSize); + UInt32 sum = 0; + for (size_t i = 0; i < kBufSize - kCheckSize * 2; i += kCheckSize - 1) + for (size_t j = 0; j < kCheckSize; j++) + { + sum = rotlFixed(sum, 11); + sum += CrcCalc(buf + i + j, j); + } + return sum == 0x28462c7c; } struct CBenchMethod @@ -2652,6 +2697,7 @@ static const CBenchMethod g_Bench[] = { 2, 0, 2, 0, 2, "BCJ" }, { 2, 0, 1, 0, 1, "ARM64" }, + { 2, 0, 1, 0, 1, "RISCV" }, // { 10, 0, 18, 0, 18, "AES128CBC:1" }, // { 10, 0, 21, 0, 21, "AES192CBC:1" }, @@ -2668,13 +2714,13 @@ static const CBenchMethod g_Bench[] = // { 2, 0, CMPLX(1), 0, CMPLX(1), "AES192CTR:2" }, // { 2, 0, CMPLX(1), 0, CMPLX(1), "AES256CTR:2" }, - // { 1, 0, CMPLX(6), 0, CMPLX(1), "AES128CBC:3" }, - // { 1, 0, CMPLX(7), 0, CMPLX(1), "AES192CBC:3" }, - { 1, 0, CMPLX(8), 0, CMPLX(1), "AES256CBC:3" } + // { 1, 0, CMPLX(6), 0, -2, "AES128CBC:3" }, + // { 1, 0, CMPLX(7), 0, -2, "AES192CBC:3" }, + { 1, 0, CMPLX(8), 0, -2, "AES256CBC:3" } - // { 1, 0, CMPLX(1), 0, CMPLX(1), "AES128CTR:3" }, - // { 1, 0, CMPLX(1), 0, CMPLX(1), "AES192CTR:3" }, - // { 1, 0, CMPLX(1), 0, CMPLX(1), "AES256CTR:3" }, + // { 1, 0, CMPLX(1), 0, -2, "AES128CTR:3" }, + // { 1, 0, CMPLX(1), 0, -2, "AES192CTR:3" }, + // { 1, 0, CMPLX(1), 0, -2, "AES256CTR:3" }, }; struct CBenchHash @@ -2692,12 +2738,11 @@ struct CBenchHash static const CBenchHash g_Hash[] = { - // { 1, 1820, 0x21e207bb, "CRC32:1" }, - // { 10, 558, 0x21e207bb, "CRC32:4" }, - { 20, 339, 0x21e207bb, "CRC32:8" } , + { 20, 256, 0x21e207bb, "CRC32:12" } , { 2, 128 *ARM_CRC_MUL, 0x21e207bb, "CRC32:32" }, { 2, 64 *ARM_CRC_MUL, 0x21e207bb, "CRC32:64" }, - { 10, 512, 0x41b901d1, "CRC64" }, + { 10, 256, 0x41b901d1, "CRC64" }, + { 10, 64, 0x43eac94f, "XXH64" }, { 10, 5100, 0x7913ba03, "SHA256:1" }, { 2, CMPLX((32 * 4 + 1) * 4 + 4), 0x7913ba03, "SHA256:2" }, @@ -2705,7 +2750,15 @@ static const CBenchHash g_Hash[] = { 10, 2340, 0xff769021, "SHA1:1" }, { 2, CMPLX((20 * 6 + 1) * 4 + 4), 0xff769021, "SHA1:2" }, - { 2, 5500, 0x85189d02, "BLAKE2sp" } + { 2, 4096, 0x85189d02, "BLAKE2sp:1" }, + { 2, 1024, 0x85189d02, "BLAKE2sp:2" }, // sse2-way4-fast + { 2, 512, 0x85189d02, "BLAKE2sp:3" } // avx2-way8-fast +#if 0 + , { 2, 2048, 0x85189d02, "BLAKE2sp:4" } // sse2-way1 + , { 2, 1024, 0x85189d02, "BLAKE2sp:5" } // sse2-way2 + , { 2, 1024, 0x85189d02, "BLAKE2sp:6" } // avx2-way2 + , { 2, 1024, 0x85189d02, "BLAKE2sp:7" } // avx2-way4 +#endif }; static void PrintNumber(IBenchPrintCallback &f, UInt64 value, unsigned size) @@ -3267,7 +3320,8 @@ HRESULT CFreqBench::FreqBench(IBenchPrintCallback *_file { progressInfoSpec.SetStartTime(); UInt32 sum = g_BenchCpuFreqTemp; - for (UInt64 k = numIterations; k > 0; k--) + UInt64 k = numIterations; + do { sum = CountCpuFreq(sum, numIterations2, g_BenchCpuFreqTemp); if (_file) @@ -3275,6 +3329,7 @@ HRESULT CFreqBench::FreqBench(IBenchPrintCallback *_file RINOK(_file->CheckBreak()) } } + while (--k); res += sum; } @@ -3406,7 +3461,7 @@ static HRESULT CrcBench( } #ifdef USE_ALLOCA - ci.AllocaSize = (i * 16 * 21) & 0x7FF; + ci.AllocaSize = BENCH_ALLOCA_VALUE(i); #endif } @@ -3627,6 +3682,7 @@ HRESULT Bench( bool multiDict, IBenchFreqCallback *freqCallback) { + // for (int y = 0; y < 10000; y++) if (!CrcInternalTest()) return E_FAIL; @@ -3850,13 +3906,13 @@ HRESULT Bench( { AString s; - #ifndef _WIN32 +#if 1 || !defined(Z7_MSC_VER_ORIGINAL) || (Z7_MSC_VER_ORIGINAL >= 1900) s += "Compiler: "; GetCompiler(s); printCallback->Print(s); printCallback->NewLine(); s.Empty(); - #endif +#endif GetSystemInfoText(s); printCallback->Print(s); @@ -3896,7 +3952,7 @@ HRESULT Bench( start = 1; const UInt64 freq = GetFreq(); // mips is constant in some compilers - const UInt64 hz = MyMultDiv64(numMilCommands * 1000000, freq, start); + const UInt64 hzVal = MyMultDiv64(numMilCommands * 1000000, freq, start); const UInt64 mipsVal = numMilCommands * freq / start; if (printCallback) { @@ -3912,7 +3968,7 @@ HRESULT Bench( } if (freqCallback) { - RINOK(freqCallback->AddCpuFreq(1, hz, kBenchmarkUsageMult)) + RINOK(freqCallback->AddCpuFreq(1, hzVal, kBenchmarkUsageMult)) } if (jj >= 1) @@ -3955,7 +4011,7 @@ HRESULT Bench( /* it can show incorrect frequency for HT threads. so we reduce freq test to (numCPUs / 2) */ - UInt32 numThreads = numThreadsSpecified >= numCPUs / 2 ? numCPUs / 2: numThreadsSpecified; + UInt32 numThreads = (numThreadsSpecified >= numCPUs / 2 ? numCPUs / 2 : numThreadsSpecified); if (numThreads < 1) numThreads = 1; diff --git a/CPP/7zip/UI/Common/CompressCall.cpp b/CPP/7zip/UI/Common/CompressCall.cpp index 42bae2a..1ded1b3 100644 --- a/CPP/7zip/UI/Common/CompressCall.cpp +++ b/CPP/7zip/UI/Common/CompressCall.cpp @@ -51,7 +51,7 @@ UString GetQuotedString(const UString &s) { UString s2 ('\"'); s2 += s; - s2 += '\"'; + s2.Add_Char('\"'); return s2; } @@ -155,14 +155,14 @@ static HRESULT CreateMap(const UStringVector &names, event.Close(); } - params += '#'; + params.Add_Char('#'); params += mappingName; - params += ':'; + params.Add_Colon(); char temp[32]; ConvertUInt64ToString(totalSize, temp); params += temp; - params += ':'; + params.Add_Colon(); params += eventName; LPVOID data = fileMapping.Map(FILE_MAP_WRITE, 0, totalSize); diff --git a/CPP/7zip/UI/Common/CompressCall2.cpp b/CPP/7zip/UI/Common/CompressCall2.cpp index 21eb472..fef0877 100644 --- a/CPP/7zip/UI/Common/CompressCall2.cpp +++ b/CPP/7zip/UI/Common/CompressCall2.cpp @@ -68,7 +68,7 @@ UString GetQuotedString(const UString &s) { UString s2 ('\"'); s2 += s; - s2 += '\"'; + s2.Add_Char('\"'); return s2; } diff --git a/CPP/7zip/UI/Common/DirItem.h b/CPP/7zip/UI/Common/DirItem.h index ae84937..3c19249 100644 --- a/CPP/7zip/UI/Common/DirItem.h +++ b/CPP/7zip/UI/Common/DirItem.h @@ -260,6 +260,8 @@ struct CDirItem: public NWindows::NFile::NFind::CFileInfoBase int OwnerGroupIndex; #endif + // bool Attrib_IsDefined; + CDirItem(): PhyParent(-1) , LogParent(-1) @@ -269,6 +271,7 @@ struct CDirItem: public NWindows::NFile::NFind::CFileInfoBase , OwnerNameIndex(-1) , OwnerGroupIndex(-1) #endif + // , Attrib_IsDefined(true) { } diff --git a/CPP/7zip/UI/Common/EnumDirItems.cpp b/CPP/7zip/UI/Common/EnumDirItems.cpp index d536c47..0758547 100644 --- a/CPP/7zip/UI/Common/EnumDirItems.cpp +++ b/CPP/7zip/UI/Common/EnumDirItems.cpp @@ -287,14 +287,12 @@ HRESULT CDirItems::EnumerateOneDir(const FString &phyPrefix, CObjectVector entries; - for (unsigned ttt = 0; ; ttt++) + for (;;) { bool found; NFind::CDirEntry de; if (!enumerator.Next(de, found)) - { return AddError(phyPrefix); - } if (!found) break; entries.Add(de); diff --git a/CPP/7zip/UI/Common/Extract.cpp b/CPP/7zip/UI/Common/Extract.cpp index 34b4871..010b01c 100644 --- a/CPP/7zip/UI/Common/Extract.cpp +++ b/CPP/7zip/UI/Common/Extract.cpp @@ -2,8 +2,6 @@ #include "StdAfx.h" -#include "../../../../C/Sort.h" - #include "../../../Common/StringConvert.h" #include "../../../Windows/FileDir.h" @@ -201,6 +199,8 @@ static HRESULT DecompressArchive( removePathParts, false, packSize); + ecs->Is_elimPrefix_Mode = elimIsPossible; + #ifdef SUPPORT_LINKS @@ -227,7 +227,13 @@ static HRESULT DecompressArchive( ConvertPropVariantToUInt64(prop, stdInProcessed); } else - result = archive->Extract(&realIndices.Front(), realIndices.Size(), testMode, ecs); + { + // v23.02: we reset completed value that could be set by Open() operation + IArchiveExtractCallback *aec = ecs; + const UInt64 val = 0; + RINOK(aec->SetCompleted(&val)) + result = archive->Extract(realIndices.ConstData(), realIndices.Size(), testMode, aec); + } const HRESULT res2 = ecsCloser.Close(); if (result == S_OK) @@ -349,11 +355,10 @@ HRESULT Extract( if (options.StdInMode) { // do we need ctime and mtime? - fi.ClearBase(); - fi.Size = 0; // (UInt64)(Int64)-1; - fi.SetAsFile(); - // NTime::GetCurUtc_FiTime(fi.MTime); - // fi.CTime = fi.ATime = fi.MTime; + // fi.ClearBase(); + // fi.Size = 0; // (UInt64)(Int64)-1; + if (!fi.SetAs_StdInFile()) + return GetLastError_noZero_HRESULT(); } else { diff --git a/CPP/7zip/UI/Common/Extract.h b/CPP/7zip/UI/Common/Extract.h index b20f607..defaa27 100644 --- a/CPP/7zip/UI/Common/Extract.h +++ b/CPP/7zip/UI/Common/Extract.h @@ -26,9 +26,10 @@ struct CExtractOptionsBase NExtract::NPathMode::EEnum PathMode; NExtract::NOverwriteMode::EEnum OverwriteMode; NExtract::NZoneIdMode::EEnum ZoneMode; + + CExtractNtOptions NtOptions; FString OutputDir; - CExtractNtOptions NtOptions; UString HashDir; CExtractOptionsBase(): diff --git a/CPP/7zip/UI/Common/HashCalc.cpp b/CPP/7zip/UI/Common/HashCalc.cpp index 94c8a06..f3d65ef 100644 --- a/CPP/7zip/UI/Common/HashCalc.cpp +++ b/CPP/7zip/UI/Common/HashCalc.cpp @@ -249,12 +249,12 @@ static void CSum_Name_OriginalToEscape(const AString &src, AString &dest) char c = src[i++]; if (c == '\n') { - dest += '\\'; + dest.Add_Char('\\'); c = 'n'; } else if (c == '\\') - dest += '\\'; - dest += c; + dest.Add_Char('\\'); + dest.Add_Char(c); } } @@ -287,7 +287,7 @@ static bool CSum_Name_EscapeToOriginal(const char *s, AString &dest) isOK = false; } } - dest += c; + dest.Add_Char(c); } return isOK; } @@ -403,7 +403,7 @@ static void WriteLine(CDynLimBuf &hashFileString, } if (isDir && !esc.IsEmpty() && esc.Back() != '/') - esc += '/'; + esc.Add_Slash(); if (tagMode) { @@ -461,8 +461,8 @@ HRESULT HashCalc( if (options.StdInMode) { CDirItem di; - di.Size = (UInt64)(Int64)-1; - di.SetAsFile(); + if (!di.SetAs_StdInFile()) + return GetLastError_noZero_HRESULT(); dirItems.Items.Add(di); } else @@ -531,7 +531,19 @@ HRESULT HashCalc( if (options.StdInMode) { +#if 1 inStream = new CStdInFileStream; +#else + if (!CreateStdInStream(inStream)) + { + const DWORD lastError = ::GetLastError(); + const HRESULT res = callback->OpenFileError(FString("stdin"), lastError); + hb.NumErrors++; + if (res != S_FALSE && res != S_OK) + return res; + continue; + } +#endif } else { @@ -561,7 +573,7 @@ HRESULT HashCalc( const FString phyPath = dirItems.GetPhyPath(i); if (!inStreamSpec->OpenShared(phyPath, options.OpenShareForWrite)) { - HRESULT res = callback->OpenFileError(phyPath, ::GetLastError()); + const HRESULT res = callback->OpenFileError(phyPath, ::GetLastError()); hb.NumErrors++; if (res != S_FALSE) return res; @@ -645,51 +657,40 @@ HRESULT HashCalc( } -static inline char GetHex_Upper(unsigned v) -{ - return (char)((v < 10) ? ('0' + v) : ('A' + (v - 10))); -} - -static inline char GetHex_Lower(unsigned v) -{ - return (char)((v < 10) ? ('0' + v) : ('a' + (v - 10))); -} - -void HashHexToString(char *dest, const Byte *data, UInt32 size) +void HashHexToString(char *dest, const Byte *data, size_t size) { - dest[size * 2] = 0; - if (!data) { - for (UInt32 i = 0; i < size; i++) + for (size_t i = 0; i < size; i++) { dest[0] = ' '; dest[1] = ' '; dest += 2; } + *dest = 0; return; } - if (size <= 8) + if (size > 8) + ConvertDataToHex_Lower(dest, data, size); + else if (size == 0) { - dest += size * 2; - for (UInt32 i = 0; i < size; i++) - { - const unsigned b = data[i]; - dest -= 2; - dest[0] = GetHex_Upper((b >> 4) & 0xF); - dest[1] = GetHex_Upper(b & 0xF); - } + *dest = 0; + return; } else { - for (UInt32 i = 0; i < size; i++) + const char *dest_start = dest; + dest += size * 2; + *dest = 0; + do { - const unsigned b = data[i]; - dest[0] = GetHex_Lower((b >> 4) & 0xF); - dest[1] = GetHex_Lower(b & 0xF); - dest += 2; + const size_t b = *data++; + dest -= 2; + dest[0] = GET_HEX_CHAR_UPPER(b >> 4); + dest[1] = GET_HEX_CHAR_UPPER(b & 15); } + while (dest != dest_start); } } @@ -720,31 +721,6 @@ void CHasherState::WriteToString(unsigned digestIndex, char *s) const namespace NHash { -static size_t ParseHexString(const char *s, Byte *dest) throw() -{ - size_t num; - for (num = 0;; num++, s += 2) - { - unsigned c = (Byte)s[0]; - unsigned v0; - if (c >= '0' && c <= '9') v0 = (c - '0'); - else if (c >= 'A' && c <= 'F') v0 = 10 + (c - 'A'); - else if (c >= 'a' && c <= 'f') v0 = 10 + (c - 'a'); - else - return num; - c = (Byte)s[1]; - unsigned v1; - if (c >= '0' && c <= '9') v1 = (c - '0'); - else if (c >= 'A' && c <= 'F') v1 = 10 + (c - 'A'); - else if (c >= 'a' && c <= 'f') v1 = 10 + (c - 'a'); - else - return num; - if (dest) - dest[num] = (Byte)(v1 | (v0 << 4)); - } -} - - #define IsWhite(c) ((c) == ' ' || (c) == '\t') bool CHashPair::IsDir() const @@ -753,7 +729,7 @@ bool CHashPair::IsDir() const return false; // here we expect that Dir items contain only zeros or no Hash for (size_t i = 0; i < Hash.Size(); i++) - if (Hash[i] != 0) + if (Hash.ConstData()[i] != 0) return false; return true; } @@ -854,10 +830,11 @@ bool CHashPair::Parse(const char *s) s++; escape = true; } + Escape = escape; // const char *kMethod = GetMethod_from_FileName(s); // if (kMethod) - if (ParseHexString(s, NULL) < 4) + if ((size_t)(FindNonHexChar(s) - s) < 4) { // BSD-style checksum line { @@ -903,10 +880,10 @@ bool CHashPair::Parse(const char *s) } { - const size_t num = ParseHexString(s, NULL); - Hash.Alloc(num); - ParseHexString(s, Hash); - const size_t numChars = num * 2; + const size_t numChars = (size_t)(FindNonHexChar(s) - s) & ~(size_t)1; + Hash.Alloc(numChars / 2); + if ((size_t)(ParseHexString(s, Hash) - Hash) != numChars / 2) + throw 101; HashString.SetFrom(s, (unsigned)numChars); s += numChars; } @@ -1091,7 +1068,7 @@ static void Add_OptSpace_String(UString &dest, const char *src) Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) { - NWindows::NCOM::CPropVariant prop; + NCOM::CPropVariant prop; switch (propID) { case kpidPhySize: if (_phySize != 0) prop = _phySize; break; @@ -1125,7 +1102,7 @@ Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) Add_OptSpace_String(s, "PGP"); if (!_pgpMethod.IsEmpty()) { - s += ":"; + s.Add_Colon(); s += _pgpMethod; } } @@ -1146,6 +1123,7 @@ Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) prop = true; break; } + default: break; } prop.Detach(value); return S_OK; @@ -1155,8 +1133,8 @@ Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) Z7_COM7F_IMF(CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *value)) { // COM_TRY_BEGIN - NWindows::NCOM::CPropVariant prop; - CHashPair &hp = HashPairs[index]; + NCOM::CPropVariant prop; + const CHashPair &hp = HashPairs[index]; switch (propID) { case kpidIsDir: @@ -1169,9 +1147,17 @@ Z7_COM7F_IMF(CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val UString path; hp.Get_UString_Path(path); - NArchive::NItemName::ReplaceToOsSlashes_Remove_TailSlash(path, - true); // useBackslashReplacement - + bool useBackslashReplacement = true; + if (_supportWindowsBackslash && !hp.Escape && path.Find(L"\\\\") < 0) + { +#if WCHAR_PATH_SEPARATOR == L'/' + path.Replace(L'\\', L'/'); +#else + useBackslashReplacement = false; +#endif + } + NArchive::NItemName::ReplaceToOsSlashes_Remove_TailSlash( + path, useBackslashReplacement); prop = path; break; } @@ -1195,6 +1181,7 @@ Z7_COM7F_IMF(CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val prop = hp.Method; break; } + default: break; } prop.Detach(value); return S_OK; @@ -1248,7 +1235,7 @@ Z7_COM7F_IMF(CHandler::Open(IInStream *stream, const UInt64 *, IArchiveOpenCallb bool cr_lf_Mode = false; { for (size_t i = 0; i < buf.Size(); i++) - if (buf[i] == 0) + if (buf.ConstData()[i] == 0) { zeroMode = true; break; @@ -1510,15 +1497,14 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, if (!buf.Alloc(kBufSize)) return E_OUTOFMEMORY; - CLocalProgress *lps = new CLocalProgress; - CMyComPtr progress = lps; + CMyComPtr2_Create lps; lps->Init(extractCallback, false); - lps->InSize = lps->OutSize = 0; - UInt32 i; - for (i = 0; i < numItems; i++) + for (UInt32 i = 0;; i++) { RINOK(lps->SetCur()) + if (i >= numItems) + break; const UInt32 index = allFilesMode ? i : indices[i]; CHashPair &hp = HashPairs[index]; @@ -1549,7 +1535,7 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, Attrib and security of output file */ askMode = NArchive::NExtract::NAskMode::kReadExternal; - extractCallback->PrepareOperation(askMode); + RINOK(extractCallback->PrepareOperation(askMode)) const bool isAltStream = false; @@ -1598,7 +1584,7 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, { if ((step & 0xFF) == 0) { - RINOK(progress->SetRatioInfo(NULL, &fileSize)) + RINOK(lps.Interface()->SetRatioInfo(NULL, &fileSize)) } UInt32 size; RINOK(inStream->Read(buf, kBufSize, &size)) @@ -1643,8 +1629,7 @@ Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, RINOK(extractCallback->SetOperationResult(opRes)) } - return lps->SetCur(); - + return S_OK; COM_TRY_END } @@ -1805,8 +1790,7 @@ Z7_COM7F_IMF(CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt RINOK(hb.SetMethods(EXTERNAL_CODECS_LOC_VARS methods)) - CLocalProgress *lps = new CLocalProgress; - CMyComPtr progress = lps; + CMyComPtr2_Create lps; lps->Init(callback, true); const UInt32 kBufSize = 1 << 15; @@ -1830,7 +1814,6 @@ Z7_COM7F_IMF(CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt if (options.HashMode_OnlyHash.Val && updateItems.Size() != 1) options.HashMode_OnlyHash.Val = false; - lps->OutSize = 0; complexity = 0; for (i = 0; i < updateItems.Size(); i++) @@ -1884,7 +1867,7 @@ Z7_COM7F_IMF(CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt if (getProps->GetProps(&size2, NULL, NULL, &mTime, NULL) == S_OK) { currentComplexity = size2; - // item.MTime = NWindows::NTime::FileTimeToUnixTime64(mTime);; + // item.MTime = NTime::FileTimeToUnixTime64(mTime);; } } */ @@ -1905,7 +1888,7 @@ Z7_COM7F_IMF(CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt { if ((step & 0xFF) == 0) { - RINOK(progress->SetRatioInfo(&fileSize, NULL)) + RINOK(lps.Interface()->SetRatioInfo(&fileSize, NULL)) // RINOK(callback->SetCompleted(&completeValue)); } UInt32 size; @@ -2027,6 +2010,9 @@ HRESULT CHandler::SetProperty(const wchar_t *nameSpec, const PROPVARIANT &value) return S_OK; } + if (name.IsEqualTo("backslash")) + return PROPVARIANT_to_bool(value, _supportWindowsBackslash); + if (name.IsPrefixedBy_Ascii_NoCase("crc")) { name.Delete(0, 3); diff --git a/CPP/7zip/UI/Common/HashCalc.h b/CPP/7zip/UI/Common/HashCalc.h index 0b527c1..1e9dbf4 100644 --- a/CPP/7zip/UI/Common/HashCalc.h +++ b/CPP/7zip/UI/Common/HashCalc.h @@ -20,7 +20,7 @@ const unsigned k_HashCalc_NumGroups = 4; if (size <= 8) : upper case : reversed byte order : it shows 32-bit/64-bit number, if data contains little-endian number if (size > 8) : lower case : original byte order (as big-endian byte sequence) */ -void HashHexToString(char *dest, const Byte *data, UInt32 size); +void HashHexToString(char *dest, const Byte *data, size_t size); enum { @@ -226,6 +226,7 @@ struct CHashPair CByteBuffer Hash; char Mode; bool IsBSD; + bool Escape; bool Size_from_Arc_Defined; bool Size_from_Disk_Defined; AString Method; @@ -259,6 +260,7 @@ struct CHashPair CHashPair(): Mode(0) , IsBSD(false) + , Escape(false) , Size_from_Arc_Defined(false) , Size_from_Disk_Defined(false) // , HashLengthInBits(0) @@ -275,6 +277,8 @@ Z7_CLASS_IMP_CHandler_IInArchive_3( ISetProperties ) bool _isArc; + bool _supportWindowsBackslash; + bool _crcSize_WasSet; UInt64 _phySize; CObjectVector HashPairs; UString _nameExtenstion; @@ -287,29 +291,26 @@ Z7_CLASS_IMP_CHandler_IInArchive_3( bool _are_there_Dirs; bool _hashSize_Defined; unsigned _hashSize; - - bool _crcSize_WasSet; UInt32 _crcSize; UStringVector _methods; + CHashOptionsLocal _options; void ClearVars(); void InitProps() { + _supportWindowsBackslash = true; _crcSize_WasSet = false; _crcSize = 4; _methods.Clear(); _options.Init_HashOptionsLocal(); } - CHashOptionsLocal _options; - bool CanUpdate() const { if (!_isArc || _is_PgpMethod || _is_CksumMode) return false; return true; - } HRESULT SetProperty(const wchar_t *nameSpec, const PROPVARIANT &value); diff --git a/CPP/7zip/UI/Common/LoadCodecs.cpp b/CPP/7zip/UI/Common/LoadCodecs.cpp index 5a65bdc..6bf53ea 100644 --- a/CPP/7zip/UI/Common/LoadCodecs.cpp +++ b/CPP/7zip/UI/Common/LoadCodecs.cpp @@ -263,6 +263,10 @@ static HRESULT GetMethodBoolProp(Func_GetMethodProperty getMethodProperty, UInt3 #pragma GCC diagnostic ignored "-Wc++98-compat-pedantic" #endif +#ifdef _WIN32 +Z7_DIAGNOSTIC_IGNORE_CAST_FUNCTION +#endif + #define MY_GET_FUNC(dest, type, lib, func) \ dest = Z7_GET_PROC_ADDRESS(type, lib.Get_HMODULE(), func); // #define MY_GET_FUNC(dest, type, func) dest = (type)(func); diff --git a/CPP/7zip/UI/Common/OpenArchive.cpp b/CPP/7zip/UI/Common/OpenArchive.cpp index a501b86..c26d4c0 100644 --- a/CPP/7zip/UI/Common/OpenArchive.cpp +++ b/CPP/7zip/UI/Common/OpenArchive.cpp @@ -350,6 +350,7 @@ Z7_COM7F_IMF(CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val case kpidMTime: if (item.FileTime_Defined) prop = item.FileTime; break; case kpidComment: if (!item.Comment.IsEmpty()) prop = item.Comment; break; case kpidType: if (!item.ArcType.IsEmpty()) prop = item.ArcType; break; + default: break; } prop.Detach(value); return S_OK; @@ -584,7 +585,7 @@ HRESULT CArc::GetItem_PathToParent(UInt32 index, UInt32 parent, UStringVector &p { { UString &s2 = parts[parts.Size() - 2]; - s2 += ':'; + s2.Add_Colon(); s2 += parts.Back(); } parts.DeleteBack(); @@ -2501,7 +2502,7 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) // printf("\nRead ask = %d", (unsigned)processedSize); UInt64 seekPos = bufPhyPos + bytesInBuf; RINOK(InStream_SeekSet(op.stream, bufPhyPos + bytesInBuf)) - RINOK(ReadStream(op.stream, byteBuffer + bytesInBuf, &processedSize)) + RINOK(ReadStream(op.stream, byteBuffer.NonConstData() + bytesInBuf, &processedSize)) // printf(" processed = %d", (unsigned)processedSize); if (processedSize == 0) { @@ -2523,7 +2524,7 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) { size_t keepSize = (size_t)(kBeforeSize - skipSize); // printf("\nmemmove skip = %d", (int)keepSize); - memmove(byteBuffer, byteBuffer + bytesInBuf - keepSize, keepSize); + memmove(byteBuffer, byteBuffer.ConstData() + bytesInBuf - keepSize, keepSize); bytesInBuf = keepSize; bufPhyPos = pos - keepSize; continue; @@ -2539,7 +2540,7 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) { size_t beg = (size_t)posInBuf - kBeforeSize; // printf("\nmemmove for after beg = %d", (int)beg); - memmove(byteBuffer, byteBuffer + beg, bytesInBuf - beg); + memmove(byteBuffer, byteBuffer.ConstData() + beg, bytesInBuf - beg); bufPhyPos += beg; bytesInBuf -= beg; continue; @@ -2597,14 +2598,14 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) scanSize++; - const Byte *buf = byteBuffer + (size_t)posInBuf; + const Byte *buf = byteBuffer.ConstData() + (size_t)posInBuf; const Byte *bufLimit = buf + scanSize; size_t ppp = 0; if (!needCheckStartOpen) { for (; buf < bufLimit && hash[HASH_VAL(buf)] == 0xFF; buf++); - ppp = (size_t)(buf - (byteBuffer + (size_t)posInBuf)); + ppp = (size_t)(buf - (byteBuffer.ConstData() + (size_t)posInBuf)); pos += ppp; if (buf == bufLimit) continue; @@ -2685,7 +2686,7 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) const size_t offsetInBuf = (size_t)(startArcPos - bufPhyPos); if (offsetInBuf < bytesInBuf) { - const UInt32 isArcRes = ai.IsArcFunc(byteBuffer + offsetInBuf, bytesInBuf - offsetInBuf); + const UInt32 isArcRes = ai.IsArcFunc(byteBuffer.ConstData() + offsetInBuf, bytesInBuf - offsetInBuf); if (isArcRes == k_IsArc_Res_NO) continue; if (isArcRes == k_IsArc_Res_NEED_MORE && endOfFile) @@ -3085,7 +3086,12 @@ HRESULT CArc::OpenStreamOrFile(COpenOptions &op) if (op.stdInMode) { +#if 1 seqStream = new CStdInFileStream; +#else + if (!CreateStdInStream(seqStream)) + return GetLastError_noZero_HRESULT(); +#endif op.seqStream = seqStream; } else if (!op.stream) @@ -3577,6 +3583,7 @@ static bool ParseTypeParams(const UString &s, COpenType &type) case 'e': type.EachPos = true; return true; case 'a': type.CanReturnArc = true; return true; case 'r': type.Recursive = true; return true; + default: break; } return false; } diff --git a/CPP/7zip/UI/Common/PropIDUtils.cpp b/CPP/7zip/UI/Common/PropIDUtils.cpp index ee9ff32..0b1357f 100644 --- a/CPP/7zip/UI/Common/PropIDUtils.cpp +++ b/CPP/7zip/UI/Common/PropIDUtils.cpp @@ -83,18 +83,26 @@ void ConvertWinAttribToString(char *s, UInt32 wa) throw() { /* some programs store posix attributes in high 16 bits. - p7zip - stores additional 0x8000 flag marker. - macos - stores additional 0x4000 flag marker. - info-zip - no additional marker. + p7zip - stores additional 0x8000 flag marker. + macos - stores additional 0x4000 flag marker. + info-zip - no additional marker. + But this code works with Attrib from internal 7zip code. + So we expect that 0x8000 marker is set, if there are posix attributes. + (DT_UNKNOWN == 0) type in high bits is possible in some case for linux files. + 0x8000 flag is possible also in ReFS (Windows)? */ - - const bool isPosix = ((wa & 0xF0000000) != 0); - + + const bool isPosix = ( + (wa & 0x8000) != 0 // FILE_ATTRIBUTE_UNIX_EXTENSION; + // && (wa & 0xFFFF0000u) != 0 + ); + UInt32 posix = 0; if (isPosix) { posix = wa >> 16; - wa &= (UInt32)0x3FFF; + if ((wa & 0xF0000000u) != 0) + wa &= (UInt32)0x3FFF; } for (unsigned i = 0; i < kNumWinAtrribFlags; i++) @@ -243,6 +251,7 @@ void ConvertPropertyToShortString2(char *dest, const PROPVARIANT &prop, PROPID p return; } */ + default: break; } ConvertPropVariantToShortString(prop, dest); @@ -262,15 +271,10 @@ void ConvertPropertyToString2(UString &dest, const PROPVARIANT &prop, PROPID pro #ifndef Z7_SFX -static inline unsigned GetHex(unsigned v) -{ - return (v < 10) ? ('0' + v) : ('A' + (v - 10)); -} - static inline void AddHexToString(AString &res, unsigned v) { - res += (char)GetHex(v >> 4); - res += (char)GetHex(v & 0xF); + res.Add_Char((char)GET_HEX_CHAR_UPPER(v >> 4)); + res.Add_Char((char)GET_HEX_CHAR_UPPER(v & 15)); } /* @@ -379,27 +383,27 @@ static const CServicesToName services_to_name[] = { { 0x38FB89B5, 0xCBC28419, 0x6D236C5C, 0x6E770057, 0x876402C0 } , "TrustedInstaller" } }; -static void ParseSid(AString &s, const Byte *p, UInt32 lim, UInt32 &sidSize) +static void ParseSid(AString &s, const Byte *p, size_t lim /* , unsigned &sidSize */) { - sidSize = 0; + // sidSize = 0; if (lim < 8) { s += "ERROR"; return; } - const UInt32 rev = p[0]; - if (rev != 1) + if (p[0] != 1) // rev { s += "UNSUPPORTED"; return; } - const UInt32 num = p[1]; - if (8 + num * 4 > lim) + const unsigned num = p[1]; + const unsigned sidSize_Loc = 8 + num * 4; + if (sidSize_Loc > lim) { s += "ERROR"; return; } - sidSize = 8 + num * 4; + // sidSize = sidSize_Loc; const UInt32 authority = GetBe32(p + 4); if (p[2] == 0 && p[3] == 0 && authority == 5 && num >= 1) @@ -462,20 +466,20 @@ static void ParseSid(AString &s, const Byte *p, UInt32 lim, UInt32 &sidSize) } } -static void ParseOwner(AString &s, const Byte *p, UInt32 size, UInt32 pos) +static void ParseOwner(AString &s, const Byte *p, size_t size, UInt32 pos) { if (pos > size) { s += "ERROR"; return; } - UInt32 sidSize = 0; - ParseSid(s, p + pos, size - pos, sidSize); + // unsigned sidSize = 0; + ParseSid(s, p + pos, size - pos /* , sidSize */); } -static void ParseAcl(AString &s, const Byte *p, UInt32 size, const char *strName, UInt32 flags, UInt32 offset) +static void ParseAcl(AString &s, const Byte *p, size_t size, const char *strName, UInt32 flags, UInt32 offset) { - const UInt32 control = Get16(p + 2); + const unsigned control = Get16(p + 2); if ((flags & control) == 0) return; const UInt32 pos = Get32(p + offset); @@ -484,7 +488,7 @@ static void ParseAcl(AString &s, const Byte *p, UInt32 size, const char *strName if (pos >= size) return; p += pos; - size -= pos; + size -= (size_t)pos; if (size < 8) return; if (Get16(p) != 2) // revision @@ -547,7 +551,7 @@ static void ParseAcl(AString &s, const Byte *p, UInt32 size, const char *strName #define MY_SE_SELF_RELATIVE (0x8000) */ -void ConvertNtSecureToString(const Byte *data, UInt32 size, AString &s) +void ConvertNtSecureToString(const Byte *data, size_t size, AString &s) { s.Empty(); if (size < 20 || size > (1 << 18)) @@ -566,30 +570,29 @@ void ConvertNtSecureToString(const Byte *data, UInt32 size, AString &s) ParseAcl(s, data, size, "s:", MY_SE_SACL_PRESENT, 12); ParseAcl(s, data, size, "d:", MY_SE_DACL_PRESENT, 16); s.Add_Space(); - s.Add_UInt32(size); - // s += '\n'; + s.Add_UInt32((UInt32)size); + // s.Add_LF(); // s += Data_To_Hex(data, size); } #ifdef _WIN32 -static bool CheckSid(const Byte *data, UInt32 size, UInt32 pos) throw() +static bool CheckSid(const Byte *data, size_t size, UInt32 pos) throw() { if (pos >= size) return false; size -= pos; if (size < 8) return false; - const UInt32 rev = data[pos]; - if (rev != 1) + if (data[pos] != 1) // rev return false; - const UInt32 num = data[pos + 1]; + const unsigned num = data[pos + 1]; return (8 + num * 4 <= size); } -static bool CheckAcl(const Byte *p, UInt32 size, UInt32 flags, UInt32 offset) throw() +static bool CheckAcl(const Byte *p, size_t size, UInt32 flags, size_t offset) throw() { - const UInt32 control = Get16(p + 2); + const unsigned control = Get16(p + 2); if ((flags & control) == 0) return true; const UInt32 pos = Get32(p + offset); @@ -599,11 +602,11 @@ static bool CheckAcl(const Byte *p, UInt32 size, UInt32 flags, UInt32 offset) th size -= pos; if (size < 8) return false; - const UInt32 aclSize = Get16(p + 2); + const unsigned aclSize = Get16(p + 2); return (aclSize <= size); } -bool CheckNtSecure(const Byte *data, UInt32 size) throw() +bool CheckNtSecure(const Byte *data, size_t size) throw() { if (size < 20) return false; @@ -653,7 +656,7 @@ static const CSecID2Name k_ReparseTags[] = { 0x80000026, "LX_BLK" } }; -bool ConvertNtReparseToString(const Byte *data, UInt32 size, UString &s) +bool ConvertNtReparseToString(const Byte *data, size_t size, UString &s) { s.Empty(); NFile::CReparseAttr attr; @@ -681,7 +684,7 @@ bool ConvertNtReparseToString(const Byte *data, UInt32 size, UString &s) if (attr.MinorError) s += " : MINOR_ERROR"; return true; - // s += " "; // for debug + // s.Add_Space(); // for debug } if (size < 8) @@ -713,7 +716,7 @@ bool ConvertNtReparseToString(const Byte *data, UInt32 size, UString &s) } } - s += ":"; + s.Add_Colon(); s.Add_UInt32(len); if (len != 0) @@ -730,8 +733,8 @@ bool ConvertNtReparseToString(const Byte *data, UInt32 size, UString &s) break; } const unsigned b = data[i]; - s += (char)GetHex((b >> 4) & 0xF); - s += (char)GetHex(b & 0xF); + s.Add_Char((char)GET_HEX_CHAR_UPPER(b >> 4)); + s.Add_Char((char)GET_HEX_CHAR_UPPER(b & 15)); } } diff --git a/CPP/7zip/UI/Common/PropIDUtils.h b/CPP/7zip/UI/Common/PropIDUtils.h index 6df1e94..dc22793 100644 --- a/CPP/7zip/UI/Common/PropIDUtils.h +++ b/CPP/7zip/UI/Common/PropIDUtils.h @@ -9,9 +9,9 @@ void ConvertPropertyToShortString2(char *dest, const PROPVARIANT &propVariant, PROPID propID, int level = 0) throw(); void ConvertPropertyToString2(UString &dest, const PROPVARIANT &propVariant, PROPID propID, int level = 0); -bool ConvertNtReparseToString(const Byte *data, UInt32 size, UString &s); -void ConvertNtSecureToString(const Byte *data, UInt32 size, AString &s); -bool CheckNtSecure(const Byte *data, UInt32 size) throw(); +bool ConvertNtReparseToString(const Byte *data, size_t size, UString &s); +void ConvertNtSecureToString(const Byte *data, size_t size, AString &s); +bool CheckNtSecure(const Byte *data, size_t size) throw(); void ConvertWinAttribToString(char *s, UInt32 wa) throw(); diff --git a/CPP/7zip/UI/Common/SetProperties.cpp b/CPP/7zip/UI/Common/SetProperties.cpp index 5e15d9c..5a05beb 100644 --- a/CPP/7zip/UI/Common/SetProperties.cpp +++ b/CPP/7zip/UI/Common/SetProperties.cpp @@ -83,6 +83,6 @@ HRESULT SetProperties(IUnknown *unknown, const CObjectVector &propert for (i = 0; i < realNames.Size(); i++) names.Add((const wchar_t *)realNames[i]); - return setProperties->SetProperties(&names.Front(), values.values, names.Size()); + return setProperties->SetProperties(names.ConstData(), values.values, names.Size()); } } diff --git a/CPP/7zip/UI/Common/Update.cpp b/CPP/7zip/UI/Common/Update.cpp index 27625ae..978630e 100644 --- a/CPP/7zip/UI/Common/Update.cpp +++ b/CPP/7zip/UI/Common/Update.cpp @@ -507,7 +507,7 @@ static HRESULT Compress( if (rp.GetNewPath(false, mainName, dest)) { needRename = true; - dest += ':'; + dest.Add_Colon(); dest += ai.Name.Ptr((unsigned)(colonPos + 1)); break; } @@ -704,7 +704,7 @@ static HRESULT Compress( } else realPath = us2fs(archivePath.GetFinalPath()); - if (outStreamSpec->Create(realPath, false)) + if (outStreamSpec->Create_NEW(realPath)) { tempFiles.Paths.Add(realPath); isOK = true; @@ -763,7 +763,7 @@ static HRESULT Compress( outStreamSpec2 = new COutFileStream; sfxOutStream = outStreamSpec2; const FString realPath = us2fs(archivePath.GetFinalPath()); - if (!outStreamSpec2->Create(realPath, false)) + if (!outStreamSpec2->Create_NEW(realPath)) return errorInfo.SetFromLastError("cannot open file", realPath); } @@ -856,7 +856,7 @@ static HRESULT Compress( if (!updateCallbackSpec->AreAllFilesClosed()) { - errorInfo.Message = "There are unclosed input file:"; + errorInfo.Message = "There are unclosed input files:"; errorInfo.FileNames = updateCallbackSpec->_openFiles_Paths; return E_FAIL; } @@ -1331,12 +1331,14 @@ HRESULT UpdateArchive( if (options.StdInMode) { CDirItem di; - di.ClearBase(); + // di.ClearBase(); + // di.Size = (UInt64)(Int64)-1; + if (!di.SetAs_StdInFile()) + return GetLastError_noZero_HRESULT(); di.Name = options.StdInFileName; - di.Size = (UInt64)(Int64)-1; - di.SetAsFile(); - NTime::GetCurUtc_FiTime(di.MTime); - di.CTime = di.ATime = di.MTime; + // di.Attrib_IsDefined = false; + // NTime::GetCurUtc_FiTime(di.MTime); + // di.CTime = di.ATime = di.MTime; dirItems.Items.Add(di); } else @@ -1626,6 +1628,8 @@ HRESULT UpdateArchive( #if defined(_WIN32) && !defined(UNDER_CE) + +Z7_DIAGNOSTIC_IGNORE_CAST_FUNCTION if (options.EMailMode) { @@ -1695,7 +1699,7 @@ HRESULT UpdateArchive( Z7_WIN_MapiMessageW m; memset(&m, 0, sizeof(m)); m.nFileCount = files.Size(); - m.lpFiles = &files.Front(); + m.lpFiles = files.NonConstData(); const UString addr (options.EMailAddress); Z7_WIN_MapiRecipDescW rec; @@ -1756,7 +1760,7 @@ HRESULT UpdateArchive( MapiMessage m; memset(&m, 0, sizeof(m)); m.nFileCount = files.Size(); - m.lpFiles = &files.Front(); + m.lpFiles = files.NonConstData(); const AString addr (GetAnsiString(options.EMailAddress)); MapiRecipDesc rec; diff --git a/CPP/7zip/UI/Common/UpdateCallback.cpp b/CPP/7zip/UI/Common/UpdateCallback.cpp index 5e2860d..d3ee639 100644 --- a/CPP/7zip/UI/Common/UpdateCallback.cpp +++ b/CPP/7zip/UI/Common/UpdateCallback.cpp @@ -7,19 +7,13 @@ #ifndef _WIN32 // #include // #include -/* -inclusion of by is deprecated since glibc 2.25. -Since glibc 2.3.3, macros have been aliases for three GNU-specific -functions: gnu_dev_makedev(), gnu_dev_major(), and gnu_dev_minor() -*/ // for major()/minor(): +#if defined(__APPLE__) || defined(__DragonFly__) || \ + defined(BSD) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) #include -#if defined(__FreeBSD__) || defined(BSD) || defined(__APPLE__) #else -#ifndef major #include #endif -#endif #endif // _WIN32 @@ -176,6 +170,7 @@ Z7_COM7F_IMF(CArchiveUpdateCallback::GetRootProp(PROPID propID, PROPVARIANT *val case kpidATime: if (ParentDirItem) PropVariant_SetFrom_FiTime(prop, ParentDirItem->ATime); break; case kpidMTime: if (ParentDirItem) PropVariant_SetFrom_FiTime(prop, ParentDirItem->MTime); break; case kpidArcFileName: if (!ArcFileName.IsEmpty()) prop = ArcFileName; break; + default: break; } prop.Detach(value); return S_OK; @@ -454,6 +449,7 @@ Z7_COM7F_IMF(CArchiveUpdateCallback::GetProperty(UInt32 index, PROPID propID, PR { case kpidSize: prop = (UInt64)0; break; case kpidIsAnti: prop = true; break; + default: break; } } else if (propID == kpidPath && up.NewNameIndex >= 0) @@ -481,8 +477,8 @@ Z7_COM7F_IMF(CArchiveUpdateCallback::GetProperty(UInt32 index, PROPID propID, PR case kpidCTime: PropVariant_SetFrom_FiTime(prop, di.CTime); break; case kpidATime: PropVariant_SetFrom_FiTime(prop, di.ATime); break; case kpidMTime: PropVariant_SetFrom_FiTime(prop, di.MTime); break; - case kpidAttrib: prop = (UInt32)di.GetWinAttrib(); break; - case kpidPosixAttrib: prop = (UInt32)di.GetPosixAttrib(); break; + case kpidAttrib: /* if (di.Attrib_IsDefined) */ prop = (UInt32)di.GetWinAttrib(); break; + case kpidPosixAttrib: /* if (di.Attrib_IsDefined) */ prop = (UInt32)di.GetPosixAttrib(); break; #if defined(_WIN32) case kpidIsAltStream: prop = di.IsAltStream; break; @@ -526,6 +522,7 @@ Z7_COM7F_IMF(CArchiveUpdateCallback::GetProperty(UInt32 index, PROPID propID, PR prop = DirItems->OwnerGroupMap.Strings[(unsigned)di.OwnerGroupIndex]; break; #endif + default: break; } } prop.Detach(value); @@ -594,8 +591,14 @@ Z7_COM7F_IMF(CArchiveUpdateCallback::GetStream2(UInt32 index, ISequentialInStrea mode != NUpdateNotifyOp::kUpdate) return S_OK; +#if 1 CStdInFileStream *inStreamSpec = new CStdInFileStream; CMyComPtr inStreamLoc(inStreamSpec); +#else + CMyComPtr inStreamLoc; + if (!CreateStdInStream(inStreamLoc)) + return GetLastError_noZero_HRESULT(); +#endif *inStream = inStreamLoc.Detach(); } else @@ -950,7 +953,7 @@ Z7_COM7F_IMF(CArchiveUpdateCallback::GetVolumeStream(UInt32 index, ISequentialOu fileName += VolExt; COutFileStream *streamSpec = new COutFileStream; CMyComPtr streamLoc(streamSpec); - if (!streamSpec->Create(fileName, false)) + if (!streamSpec->Create_NEW(fileName)) return GetLastError_noZero_HRESULT(); *volumeStream = streamLoc.Detach(); return S_OK; diff --git a/CPP/7zip/UI/Common/UpdateProduce.cpp b/CPP/7zip/UI/Common/UpdateProduce.cpp index e921dc3..88d2501 100644 --- a/CPP/7zip/UI/Common/UpdateProduce.cpp +++ b/CPP/7zip/UI/Common/UpdateProduce.cpp @@ -24,7 +24,7 @@ void UpdateProduce( up2.NewData = up2.NewProps = true; up2.UseArcProps = false; - switch (actionSet.StateActions[(unsigned)pair.State]) + switch ((int)actionSet.StateActions[(unsigned)pair.State]) { case NPairAction::kIgnore: if (pair.ArcIndex >= 0 && callback) @@ -61,6 +61,8 @@ void UpdateProduce( up2.IsAnti = true; up2.UseArcProps = (pair.ArcIndex >= 0); break; + + default: throw 123; // break; // is unexpected case } up2.IsSameTime = ((unsigned)pair.State == NUpdateArchive::NPairState::kSameFiles); diff --git a/CPP/7zip/UI/Common/ZipRegistry.cpp b/CPP/7zip/UI/Common/ZipRegistry.cpp index 6c1b9c8..73c56cf 100644 --- a/CPP/7zip/UI/Common/ZipRegistry.cpp +++ b/CPP/7zip/UI/Common/ZipRegistry.cpp @@ -98,6 +98,7 @@ static LPCTSTR const kSplitDest = TEXT("SplitDest"); static LPCTSTR const kElimDup = TEXT("ElimDup"); // static LPCTSTR const kAltStreams = TEXT("AltStreams"); static LPCTSTR const kNtSecur = TEXT("Security"); +static LPCTSTR const kMemLimit = TEXT("MemLimit"); void CInfo::Save() const { @@ -128,6 +129,14 @@ void Save_ShowPassword(bool showPassword) key.SetValue(kShowPassword, showPassword); } +void Save_LimitGB(UInt32 limit_GB) +{ + CS_LOCK + CKey key; + CreateMainKey(key, kKeyName); + Key_Set_UInt32(key, kMemLimit, limit_GB); +} + void CInfo::Load() { PathMode = NPathMode::kCurPaths; @@ -176,6 +185,19 @@ bool Read_ShowPassword() return showPassword; } +UInt32 Read_LimitGB() +{ + CS_LOCK + CKey key; + if (OpenMainKey(key, kKeyName) == ERROR_SUCCESS) + { + UInt32 v; + if (key.QueryValue(kMemLimit, v) == ERROR_SUCCESS) + return v; + } + return (UInt32)(Int32)-1; +} + } namespace NCompression diff --git a/CPP/7zip/UI/Common/ZipRegistry.h b/CPP/7zip/UI/Common/ZipRegistry.h index 6bc6977..ce084f4 100644 --- a/CPP/7zip/UI/Common/ZipRegistry.h +++ b/CPP/7zip/UI/Common/ZipRegistry.h @@ -43,6 +43,9 @@ namespace NExtract void Save_ShowPassword(bool showPassword); bool Read_ShowPassword(); + + void Save_LimitGB(UInt32 limit_GB); + UInt32 Read_LimitGB(); } namespace NCompression diff --git a/CPP/7zip/UI/Console/Console.dsp b/CPP/7zip/UI/Console/Console.dsp index 26e8c7f..0ca250d 100644 --- a/CPP/7zip/UI/Console/Console.dsp +++ b/CPP/7zip/UI/Console/Console.dsp @@ -397,6 +397,10 @@ SOURCE=..\..\..\Common\Common.h # End Source File # Begin Source File +SOURCE=..\..\..\Common\Common0.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Common\ComTry.h # End Source File # Begin Source File @@ -869,6 +873,10 @@ SOURCE=..\..\..\..\C\7zTypes.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\7zVersion.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\7zWindows.h # End Source File # Begin Source File @@ -904,6 +912,10 @@ SOURCE=..\..\..\..\C\DllSecur.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\Precomp.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Sort.c # SUBTRACT CPP /YX /Yc /Yu # End Source File diff --git a/CPP/7zip/UI/Console/Console.mak b/CPP/7zip/UI/Console/Console.mak index 1a47bfa..4effea6 100644 --- a/CPP/7zip/UI/Console/Console.mak +++ b/CPP/7zip/UI/Console/Console.mak @@ -1,7 +1,8 @@ MY_CONSOLE = 1 !IFNDEF UNDER_CE -CFLAGS = $(CFLAGS) -DZ7_LONG_PATH -DZ7_LARGE_PAGES -DZ7_DEVICE_FILE +CFLAGS = $(CFLAGS) -DZ7_DEVICE_FILE +# -DZ7_LONG_PATH -DZ7_LARGE_PAGES !ENDIF CONSOLE_OBJS = \ diff --git a/CPP/7zip/UI/Console/ExtractCallbackConsole.cpp b/CPP/7zip/UI/Console/ExtractCallbackConsole.cpp index dd7a214..66d7123 100644 --- a/CPP/7zip/UI/Console/ExtractCallbackConsole.cpp +++ b/CPP/7zip/UI/Console/ExtractCallbackConsole.cpp @@ -64,7 +64,7 @@ HRESULT CExtractScanConsole::ScanError(const FString &path, DWORD systemError) if (_se) { *_se << endl << kError << NError::MyFormatMessage(systemError) << endl; - _se->NormalizePrint_UString(fs2us(path)); + _se->NormalizePrint_UString_Path(fs2us(path)); *_se << endl << endl; _se->Flush(); } @@ -102,7 +102,7 @@ void PrintSize_bytes_Smart(AString &s, UInt64 val) temp[0] = c; s += " ("; Print_UInt64_and_String(s, ((val + ((UInt64)1 << numBits) - 1) >> numBits), temp); - s += ')'; + s.Add_Char(')'); } static void PrintSize_bytes_Smart_comma(AString &s, UInt64 val) @@ -265,7 +265,7 @@ static const char * const kTab = " "; static void PrintFileInfo(CStdOutStream *_so, const wchar_t *path, const FILETIME *ft, const UInt64 *size) { *_so << kTab << "Path: "; - _so->NormalizePrint_wstr(path); + _so->NormalizePrint_wstr_Path(path); *_so << endl; if (size && *size != (UInt64)(Int64)-1) { @@ -358,7 +358,7 @@ Z7_COM7F_IMF(CExtractCallbackConsole::PrepareOperation(const wchar_t *name, Int3 if (name) { _tempU = name; - _so->Normalize_UString(_tempU); + _so->Normalize_UString_Path(_tempU); // 21.04 if (isFolder) { @@ -448,6 +448,7 @@ void SetExtractErrorMessage(Int32 opRes, Int32 encrypted, AString &dest) case NArchive::NExtract::NOperationResult::kWrongPassword: s = kWrongPassword; break; + default: break; } dest += kError; @@ -489,7 +490,7 @@ Z7_COM7F_IMF(CExtractCallbackConsole::SetOperationResult(Int32 opRes, Int32 encr if (!_currentName.IsEmpty()) { *_se << " : "; - _se->NormalizePrint_UString(_currentName); + _se->NormalizePrint_UString_Path(_currentName); } *_se << endl; _se->Flush(); @@ -531,8 +532,100 @@ Z7_COM7F_IMF(CExtractCallbackConsole::CryptoGetTextPassword(BSTR *password)) #endif + +#ifndef Z7_SFX + +void CExtractCallbackConsole::PrintTo_se_Path_WithTitle(const UString &path, const char *title) +{ + *_se << title; + _se->NormalizePrint_UString_Path(path); + *_se << endl; +} + +void CExtractCallbackConsole::Add_ArchiveName_Error() +{ + if (_needWriteArchivePath) + { + PrintTo_se_Path_WithTitle(_currentArchivePath, "Archive: "); + _needWriteArchivePath = false; + } +} + + +Z7_COM7F_IMF(CExtractCallbackConsole::RequestMemoryUse( + UInt32 flags, UInt32 /* indexType */, UInt32 /* index */, const wchar_t *path, + UInt64 requiredSize, UInt64 *allowedSize, UInt32 *answerFlags)) +{ + if ((flags & NRequestMemoryUseFlags::k_IsReport) == 0 + && requiredSize <= *allowedSize) + { +#if 0 + // it's expected, that *answerFlags was set to NRequestMemoryAnswerFlags::k_Allow already, + // because it's default answer for (requiredSize <= *allowedSize) case. + // optional code: + *answerFlags = NRequestMemoryAnswerFlags::k_Allow; +#endif + } + else + { + if ((flags & NRequestMemoryUseFlags::k_NoErrorMessage) == 0) + if (_se) + { + const UInt64 num_GB_allowed = (*allowedSize + ((1u << 30) - 1)) >> 30; + const UInt64 num_GB_required = (requiredSize + ((1u << 30) - 1)) >> 30; + ClosePercentsAndFlush(); + Add_ArchiveName_Error(); + if (path) + PrintTo_se_Path_WithTitle(path, "File: "); + *_se << "The extraction operation requires big amount memory (RAM):" << endl + << " " << num_GB_required << " GB : required memory usage size" << endl + << " " << num_GB_allowed << " GB : allowed memory usage limit" << endl + << " Use -smemx{size}g switch to set allowed memory usage limit for extraction." << endl; + *_se << "ERROR: Memory usage limit was exceeded." << endl; + const char *m = NULL; + // if (indexType == NArchive::NEventIndexType::kNoIndex) + if ((flags & NRequestMemoryUseFlags::k_SkipArc_IsExpected) || + (flags & NRequestMemoryUseFlags::k_Report_SkipArc)) + m = "Archive unpacking was skipped."; +/* + else if ((flags & NRequestMemoryUseFlags::k_SkipBigFiles_IsExpected) || + (flags & NRequestMemoryUseFlags::k_Report_SkipBigFiles)) + m = "Extraction for some files will be skipped."; + else if ((flags & NRequestMemoryUseFlags::k_SkipBigFile_IsExpected) || + (flags & NRequestMemoryUseFlags::k_Report_SkipBigFile)) + m = "File extraction was skipped."; +*/ + if (m) + *_se << m; + _se->Flush(); + } + + if ((flags & NRequestMemoryUseFlags::k_IsReport) == 0) + { + // default answer can be k_Allow, if limit was not forced, + // so we change answer to non-allowed here. + *answerFlags = NRequestMemoryAnswerFlags::k_Limit_Exceeded; + if (flags & NRequestMemoryUseFlags::k_SkipArc_IsExpected) + *answerFlags |= NRequestMemoryAnswerFlags::k_SkipArc; +/* + else if (flags & NRequestMemoryUseFlags::k_SkipBigFile_IsExpected) + *answerFlags |= NRequestMemoryAnswerFlags::k_SkipBigFile; + else if (flags & NRequestMemoryUseFlags::k_SkipBigFiles_IsExpected) + *answerFlags |= NRequestMemoryAnswerFlags::k_SkipBigFiles; +*/ + } + } + return CheckBreak2(); +} + +#endif + + HRESULT CExtractCallbackConsole::BeforeOpen(const wchar_t *name, bool testMode) { + _currentArchivePath = name; + _needWriteArchivePath = true; + RINOK(CheckBreak2()) NumTryArcs++; @@ -544,7 +637,7 @@ HRESULT CExtractCallbackConsole::BeforeOpen(const wchar_t *name, bool testMode) if (_so) { *_so << endl << (testMode ? kTesting : kExtracting); - _so->NormalizePrint_wstr(name); + _so->NormalizePrint_wstr_Path(name); *_so << endl; } @@ -609,7 +702,7 @@ void Print_ErrorFormatIndex_Warning(CStdOutStream *_so, const CCodecs *codecs, c const CArcErrorInfo &er = arc.ErrorInfo; *_so << "WARNING:\n"; - _so->NormalizePrint_UString(arc.Path); + _so->NormalizePrint_UString_Path(arc.Path); UString s; if (arc.FormatIndex == er.ErrorFormatIndex) { @@ -630,6 +723,9 @@ HRESULT CExtractCallbackConsole::OpenResult( const CCodecs *codecs, const CArchiveLink &arcLink, const wchar_t *name, HRESULT result) { + _currentArchivePath = name; + _needWriteArchivePath = true; + ClosePercents(); if (NeedPercents()) @@ -656,7 +752,7 @@ HRESULT CExtractCallbackConsole::OpenResult( *_se << endl; if (level != 0) { - _se->NormalizePrint_UString(arc.Path); + _se->NormalizePrint_UString_Path(arc.Path); *_se << endl; } } @@ -693,7 +789,7 @@ HRESULT CExtractCallbackConsole::OpenResult( *_so << endl; if (level != 0) { - _so->NormalizePrint_UString(arc.Path); + _so->NormalizePrint_UString_Path(arc.Path); *_so << endl; } } @@ -751,7 +847,7 @@ HRESULT CExtractCallbackConsole::OpenResult( if (_se) { *_se << kError; - _se->NormalizePrint_wstr(name); + _se->NormalizePrint_wstr_Path(name); *_se << endl; const HRESULT res = Print_OpenArchive_Error(*_se, codecs, arcLink); RINOK(res) diff --git a/CPP/7zip/UI/Console/ExtractCallbackConsole.h b/CPP/7zip/UI/Console/ExtractCallbackConsole.h index 478b293..8db80e7 100644 --- a/CPP/7zip/UI/Console/ExtractCallbackConsole.h +++ b/CPP/7zip/UI/Console/ExtractCallbackConsole.h @@ -87,6 +87,10 @@ class CExtractCallbackConsole Z7_final: #ifndef Z7_NO_CRYPTO public ICryptoGetTextPassword, #endif + #ifndef Z7_SFX + public IArchiveRequestMemoryUseCallback, + #endif + public COpenCallbackConsole, public CMyUnknownImp { @@ -96,6 +100,10 @@ class CExtractCallbackConsole Z7_final: #ifndef Z7_NO_CRYPTO Z7_COM_QI_ENTRY(ICryptoGetTextPassword) #endif + #ifndef Z7_SFX + Z7_COM_QI_ENTRY(IArchiveRequestMemoryUseCallback) + #endif + Z7_COM_QI_END Z7_COM_ADDREF_RELEASE @@ -107,13 +115,29 @@ class CExtractCallbackConsole Z7_final: #ifndef Z7_NO_CRYPTO Z7_IFACE_COM7_IMP(ICryptoGetTextPassword) #endif - + #ifndef Z7_SFX + Z7_IFACE_COM7_IMP(IArchiveRequestMemoryUseCallback) + #endif + + bool _needWriteArchivePath; + +public: + bool ThereIsError_in_Current; + bool ThereIsWarning_in_Current; + bool NeedFlush; +private: AString _tempA; UString _tempU; + UString _currentArchivePath; UString _currentName; +#ifndef Z7_SFX + void PrintTo_se_Path_WithTitle(const UString &path, const char *title); + void Add_ArchiveName_Error(); +#endif + void ClosePercents_for_so() { if (NeedPercents() && _so == _percent._so) @@ -130,9 +154,6 @@ class CExtractCallbackConsole Z7_final: public: UInt64 NumTryArcs; - bool ThereIsError_in_Current; - bool ThereIsWarning_in_Current; - UInt64 NumOkArcs; UInt64 NumCantOpenArcs; UInt64 NumArcsWithError; @@ -144,11 +165,11 @@ class CExtractCallbackConsole Z7_final: UInt64 NumFileErrors; UInt64 NumFileErrors_in_Current; - bool NeedFlush; unsigned PercentsNameLevel; unsigned LogLevel; CExtractCallbackConsole(): + _needWriteArchivePath(true), NeedFlush(false), PercentsNameLevel(1), LogLevel(0) diff --git a/CPP/7zip/UI/Console/HashCon.cpp b/CPP/7zip/UI/Console/HashCon.cpp index c0e69e2..078a625 100644 --- a/CPP/7zip/UI/Console/HashCon.cpp +++ b/CPP/7zip/UI/Console/HashCon.cpp @@ -314,7 +314,7 @@ HRESULT CHashCallbackConsole::SetOperationResult(UInt64 fileSize, const CHashBun else { UString temp (_fileName); - _so->Normalize_UString(temp); + _so->Normalize_UString_Path(temp); _so->Convert_UString_to_AString(temp, s); } PrintResultLine(fileSize, hb.Hashers, k_HashCalc_Index_Current, showHash, s); diff --git a/CPP/7zip/UI/Console/List.cpp b/CPP/7zip/UI/Console/List.cpp index 46819f1..3f33dac 100644 --- a/CPP/7zip/UI/Console/List.cpp +++ b/CPP/7zip/UI/Console/List.cpp @@ -240,11 +240,13 @@ static void PrintUString(EAdjustment adj, unsigned width, const UString &s, AStr { numSpaces = width - s.Len(); unsigned numLeftSpaces = 0; - switch (adj) + switch ((int)adj) { - case kLeft: numLeftSpaces = 0; break; + // case kLeft: numLeftSpaces = 0; break; case kCenter: numLeftSpaces = numSpaces / 2; break; case kRight: numLeftSpaces = numSpaces; break; + // case kLeft: + default: break; } PrintSpaces(numLeftSpaces); numSpaces -= numLeftSpaces; @@ -263,11 +265,13 @@ static void PrintString(EAdjustment adj, unsigned width, const char *s) { numSpaces = width - len; unsigned numLeftSpaces = 0; - switch (adj) + switch ((int)adj) { - case kLeft: numLeftSpaces = 0; break; + // case kLeft: numLeftSpaces = 0; break; case kCenter: numLeftSpaces = numSpaces / 2; break; case kRight: numLeftSpaces = numSpaces; break; + // case kLeft: + default: break; } PrintSpaces(numLeftSpaces); numSpaces -= numLeftSpaces; @@ -286,11 +290,13 @@ static void PrintStringToString(char *dest, EAdjustment adj, unsigned width, con { numSpaces = width - len; unsigned numLeftSpaces = 0; - switch (adj) + switch ((int)adj) { - case kLeft: numLeftSpaces = 0; break; + // case kLeft: numLeftSpaces = 0; break; case kCenter: numLeftSpaces = numSpaces / 2; break; case kRight: numLeftSpaces = numSpaces; break; + // case kLeft: + default: break; } PrintSpacesToString(dest, numLeftSpaces); dest += numLeftSpaces; @@ -443,10 +449,10 @@ void CFieldPrinter::AddProp(const wchar_t *name, PROPID propID, bool isRawProp) unsigned i; for (i = 0; i < s.Len(); i++) { - wchar_t c = s[i]; + const wchar_t c = s[i]; if (c >= 0x80) break; - sA += (char)c; + sA.Add_Char((char)c); } if (i == s.Len()) f.NameA = sA; @@ -504,7 +510,8 @@ static void PrintTime(char *dest, const CListFileTimeDef &t, bool showNS) if (t.IsZero()) return; int prec = kTimestampPrintLevel_SEC; - if (showNS) + unsigned flags = 0; + if (showNS) // techmode { prec = kTimestampPrintLevel_NTFS; if (t.Prec != 0) @@ -514,33 +521,30 @@ static void PrintTime(char *dest, const CListFileTimeDef &t, bool showNS) prec = kTimestampPrintLevel_NTFS; } } + else + { + // we want same default number of characters, so we disable 'Z' marker: + flags = kTimestampPrintFlags_DisableZ; + } - ConvertUtcFileTimeToString2(t.FT, t.Ns100, dest, prec); + ConvertUtcFileTimeToString2(t.FT, t.Ns100, dest, prec, flags); } #ifndef Z7_SFX -static inline char GetHex(Byte value) -{ - return (char)((value < 10) ? ('0' + value) : ('a' + (value - 10))); -} - -static void HexToString(char *dest, const Byte *data, UInt32 size) -{ - for (UInt32 i = 0; i < size; i++) - { - Byte b = data[i]; - dest[0] = GetHex((Byte)((b >> 4) & 0xF)); - dest[1] = GetHex((Byte)(b & 0xF)); - dest += 2; - } - *dest = 0; -} - #endif #define MY_ENDL endl +inline bool IsPropId_for_PathString(UInt32 propId) +{ + return (propId == kpidPath + // || propId == kpidName + || propId == kpidSymLink + || propId == kpidHardLink + || propId == kpidCopyLink); +} + HRESULT CFieldPrinter::PrintItemInfo(UInt32 index, const CListStat &st) { char temp[128]; @@ -577,7 +581,7 @@ HRESULT CFieldPrinter::PrintItemInfo(UInt32 index, const CListStat &st) { if (!techMode) g_StdOut << temp; - g_StdOut.NormalizePrint_UString(FilePath, TempWString, TempAString); + g_StdOut.NormalizePrint_UString_Path(FilePath, TempWString, TempAString); if (techMode) g_StdOut << MY_ENDL; continue; @@ -633,7 +637,7 @@ HRESULT CFieldPrinter::PrintItemInfo(UInt32 index, const CListStat &st) else { char hexStr[kMaxDataSize * 2 + 4]; - HexToString(hexStr, (const Byte *)data, dataSize); + ConvertDataToHex_Lower(hexStr, (const Byte *)data, dataSize); g_StdOut << hexStr; } } @@ -697,11 +701,12 @@ HRESULT CFieldPrinter::PrintItemInfo(UInt32 index, const CListStat &st) { TempWString.SetFromBstr(prop.bstrVal); // do we need multi-line support here ? - g_StdOut.Normalize_UString(TempWString); + if (IsPropId_for_PathString(f.PropID)) + g_StdOut.Normalize_UString_Path(TempWString); + else + g_StdOut.Normalize_UString(TempWString); if (techMode) - { g_StdOut.PrintUString(TempWString, TempAString); - } else PrintUString(f.TextAdjustment, width, TempWString, TempAString); } @@ -857,10 +862,9 @@ static void UString_Replace_CRLF_to_LF(UString &s) } *dest++ = c; } - s.ReleaseBuf_SetEnd((unsigned)(dest - s.GetBuf())); + s.ReleaseBuf_SetEnd((unsigned)(size_t)(dest - s.GetBuf())); } - static void PrintPropVal_MultiLine(CStdOutStream &so, const wchar_t *val) { UString s (val); @@ -870,21 +874,34 @@ static void PrintPropVal_MultiLine(CStdOutStream &so, const wchar_t *val) so << "{"; so << endl; UString_Replace_CRLF_to_LF(s); - so.Normalize_UString_LF_Allowed(s); - so << s; - so << endl; + UString temp; + unsigned start = 0; + for (;;) + { + unsigned size = s.Len() - start; + if (size == 0) + break; + const int next = s.Find(L'\n', start); + if (next >= 0) + size = (unsigned)next - start; + temp.SetFrom(s.Ptr() + start, size); + so.NormalizePrint_UString(temp); + so << endl; + if (next < 0) + break; + start = (unsigned)next + 1; + } so << "}"; } else { - so.Normalize_UString(s); - so << s; + so.NormalizePrint_UString(s); } so << endl; } -static void PrintPropPair(CStdOutStream &so, const char *name, const wchar_t *val, bool multiLine) +static void PrintPropPair(CStdOutStream &so, const char *name, const wchar_t *val, bool multiLine, bool isPath = false) { so << name << " = "; if (multiLine) @@ -893,12 +910,22 @@ static void PrintPropPair(CStdOutStream &so, const char *name, const wchar_t *va return; } UString s (val); - so.Normalize_UString(s); + if (isPath) + so.Normalize_UString_Path(s); + else + so.Normalize_UString(s); so << s; so << endl; } +static void PrintPropPair_Path(CStdOutStream &so, const wchar_t *path) +{ + PrintPropPair(so, "Path", path, + false, // multiLine + true); // isPath +} + static void PrintPropertyPair2(CStdOutStream &so, PROPID propID, const wchar_t *name, const CPropVariant &prop) { UString s; @@ -959,7 +986,7 @@ HRESULT Print_OpenArchive_Props(CStdOutStream &so, const CCodecs *codecs, const const CArcErrorInfo &er = arc.ErrorInfo; so << "--\n"; - PrintPropPair(so, "Path", arc.Path, false); + PrintPropPair_Path(so, arc.Path); if (er.ErrorFormatIndex >= 0) { if (er.ErrorFormatIndex == arc.FormatIndex) @@ -1026,7 +1053,7 @@ HRESULT Print_OpenArchive_Error(CStdOutStream &so, const CCodecs *codecs, const { if (arcLink.NonOpen_ErrorInfo.ErrorFormatIndex >= 0) { - so.NormalizePrint_UString(arcLink.NonOpen_ArcPath); + so.NormalizePrint_UString_Path(arcLink.NonOpen_ArcPath); so << endl; PrintArcTypeError(so, codecs->Formats[(unsigned)arcLink.NonOpen_ErrorInfo.ErrorFormatIndex].Name, false); } @@ -1103,7 +1130,7 @@ HRESULT ListArchives( if (g_ErrStream) { *g_ErrStream << endl << kError << NError::MyFormatMessage(errorCode) << endl; - g_ErrStream->NormalizePrint_UString(arcPath); + g_ErrStream->NormalizePrint_UString_Path(arcPath); *g_ErrStream << endl << endl; } numErrors++; @@ -1115,7 +1142,7 @@ HRESULT ListArchives( if (g_ErrStream) { *g_ErrStream << endl << kError; - g_ErrStream->NormalizePrint_UString(arcPath); + g_ErrStream->NormalizePrint_UString_Path(arcPath); *g_ErrStream << " is not a file" << endl << endl; } numErrors++; @@ -1157,7 +1184,7 @@ HRESULT ListArchives( if (enableHeaders) { g_StdOut << endl << kListing; - g_StdOut.NormalizePrint_UString(arcPath); + g_StdOut.NormalizePrint_UString_Path(arcPath); g_StdOut << endl << endl; } @@ -1173,7 +1200,7 @@ HRESULT ListArchives( if (g_ErrStream) { *g_ErrStream << endl << kError; - g_ErrStream->NormalizePrint_UString(arcPath); + g_ErrStream->NormalizePrint_UString_Path(arcPath); *g_ErrStream << " : "; if (result == S_FALSE) { @@ -1361,7 +1388,7 @@ HRESULT ListArchives( if (arcLink.NonOpen_ErrorInfo.ErrorFormatIndex >= 0) { g_StdOut << "----------\n"; - PrintPropPair(g_StdOut, "Path", arcLink.NonOpen_ArcPath, false); + PrintPropPair_Path(g_StdOut, arcLink.NonOpen_ArcPath); PrintArcTypeError(g_StdOut, codecs->Formats[(unsigned)arcLink.NonOpen_ErrorInfo.ErrorFormatIndex].Name, false); } } diff --git a/CPP/7zip/UI/Console/Main.cpp b/CPP/7zip/UI/Console/Main.cpp index eb9e2a8..854e110 100644 --- a/CPP/7zip/UI/Console/Main.cpp +++ b/CPP/7zip/UI/Console/Main.cpp @@ -16,7 +16,7 @@ #else // Z7_OLD_WIN_SDK -typedef struct _PROCESS_MEMORY_COUNTERS { +typedef struct { DWORD cb; DWORD PageFaultCount; SIZE_T PeakWorkingSetSize; @@ -114,7 +114,7 @@ DECLARE_AND_SET_CLIENT_VERSION_VAR #elif defined(Z7_PROG_VARIANT_R) #define PROG_POSTFIX "r" #define PROG_POSTFIX_2 " (r)" -#elif !defined(Z7_EXTERNAL_CODECS) +#elif defined(Z7_PROG_VARIANT_A) || !defined(Z7_EXTERNAL_CODECS) #define PROG_POSTFIX "a" #define PROG_POSTFIX_2 " (a)" #else @@ -148,15 +148,15 @@ static const char * const kHelpString = "\n" "\n" " -- : Stop switches and @listfile parsing\n" - " -ai[r[-|0]]{@listfile|!wildcard} : Include archives\n" - " -ax[r[-|0]]{@listfile|!wildcard} : eXclude archives\n" + " -ai[r[-|0]][m[-|2]][w[-]]{@listfile|!wildcard} : Include archives\n" + " -ax[r[-|0]][m[-|2]][w[-]]{@listfile|!wildcard} : eXclude archives\n" " -ao{a|s|t|u} : set Overwrite mode\n" " -an : disable archive_name field\n" " -bb[0-3] : set output log level\n" " -bd : disable progress indicator\n" " -bs{o|e|p}{0|1|2} : set output stream for output/error/progress line\n" " -bt : show execution time statistics\n" - " -i[r[-|0]]{@listfile|!wildcard} : Include filenames\n" + " -i[r[-|0]][m[-|2]][w[-]]{@listfile|!wildcard} : Include filenames\n" " -m{Parameters} : set compression Method\n" " -mmt[N] : set number of CPU threads\n" " -mx[N] : set compression level: -mx1 (fastest) ... -mx9 (ultra)\n" @@ -168,7 +168,14 @@ static const char * const kHelpString = " -sa{a|e|s} : set Archive name mode\n" " -scc{UTF-8|WIN|DOS} : set charset for console input/output\n" " -scs{UTF-8|UTF-16LE|UTF-16BE|WIN|DOS|{id}} : set charset for list files\n" - " -scrc[CRC32|CRC64|SHA1|SHA256|*] : set hash function for x, e, h commands\n" + " -scrc[CRC32|CRC64|SHA256" +#ifndef Z7_PROG_VARIANT_R + "|SHA1|XXH64" +#ifdef Z7_PROG_VARIANT_Z + "|BLAKE2SP" +#endif +#endif + "|*] : set hash function for x, e, h commands\n" " -sdel : delete files after compression\n" " -seml[.] : send archive by email\n" " -sfx[{name}] : Create SFX archive\n" @@ -194,7 +201,7 @@ static const char * const kHelpString = " -u[-][p#][q#][r#][x#][y#][z#][!newArchiveName] : Update options\n" " -v{Size}[b|k|m|g] : Create volumes\n" " -w[{path}] : assign Work directory. Empty path means a temporary directory\n" - " -x[r[-|0]]{@listfile|!wildcard} : eXclude filenames\n" + " -x[r[-|0]][m[-|2]][w[-]]{@listfile|!wildcard} : eXclude filenames\n" " -y : assume Yes on all queries\n"; // --------------------------- @@ -206,7 +213,9 @@ static const char * const kNoFormats = "7-Zip cannot find the code that works wi static const char * const kUnsupportedArcTypeMessage = "Unsupported archive type"; // static const char * const kUnsupportedUpdateArcType = "Can't create archive for that type"; +#ifndef Z7_EXTRACT_ONLY #define kDefaultSfxModule "7zCon.sfx" +#endif Z7_ATTR_NORETURN static void ShowMessageAndThrowException(LPCSTR message, NExitCode::EEnum code) @@ -244,10 +253,50 @@ static void ShowProgInfo(CStdOutStream *so) #ifdef __ARM_ARCH << " arm_v:" << __ARM_ARCH + #if (__ARM_ARCH == 8) + // for macos: + #if defined(__ARM_ARCH_8_9__) + << ".9" + #elif defined(__ARM_ARCH_8_8__) + << ".8" + #elif defined(__ARM_ARCH_8_7__) + << ".7" + #elif defined(__ARM_ARCH_8_6__) + << ".6" + #elif defined(__ARM_ARCH_8_5__) + << ".5" + #elif defined(__ARM_ARCH_8_4__) + << ".4" + #elif defined(__ARM_ARCH_8_3__) + << ".3" + #elif defined(__ARM_ARCH_8_2__) + << ".2" + #elif defined(__ARM_ARCH_8_1__) + << ".1" + #endif + #endif + + #if defined(__ARM_ARCH_PROFILE) && \ + ( __ARM_ARCH_PROFILE >= 'A' && __ARM_ARCH_PROFILE <= 'Z' \ + || __ARM_ARCH_PROFILE >= 65 && __ARM_ARCH_PROFILE <= 65 + 25) + << "-" << (char)__ARM_ARCH_PROFILE + #endif + #ifdef __ARM_ARCH_ISA_THUMB << " thumb:" << __ARM_ARCH_ISA_THUMB #endif #endif + + #ifdef _MIPS_ARCH + << " mips_arch:" << _MIPS_ARCH + #endif + #ifdef __mips_isa_rev + << " mips_isa_rev:" << __mips_isa_rev + #endif + + #ifdef __iset__ + << " e2k_v:" << __iset__ + #endif ; @@ -380,16 +429,11 @@ static void PrintString(CStdOutStream &so, const UString &s, unsigned size) so << ' '; } -static inline char GetHex(unsigned val) -{ - return (char)((val < 10) ? ('0' + val) : ('A' + (val - 10))); -} - static void PrintWarningsPaths(const CErrorPathCodes &pc, CStdOutStream &so) { FOR_VECTOR(i, pc.Paths) { - so.NormalizePrint_UString(fs2us(pc.Paths[i])); + so.NormalizePrint_UString_Path(fs2us(pc.Paths[i])); so << " : "; so << NError::MyFormatMessage(pc.Codes[i]) << endl; } @@ -570,6 +614,8 @@ static void PrintStat() #ifndef UNDER_CE +Z7_DIAGNOSTIC_IGNORE_CAST_FUNCTION + PROCESS_MEMORY_COUNTERS m; memset(&m, 0, sizeof(m)); BOOL memDefined = FALSE; @@ -811,11 +857,11 @@ int Main2( for (int i = 0; i < numArgs; i++) { AString a (args[i]); - /* +#if 0 printf("\n%d %s :", i, a.Ptr()); for (unsigned k = 0; k < a.Len(); k++) printf(" %2x", (unsigned)(Byte)a[k]); - */ +#endif const UString s = MultiByteToUnicodeString(a); commandStrings.Add(s); } @@ -907,6 +953,8 @@ int Main2( if (stderr_cp != -1) g_StdErr.CodePage = stderr_cp; if (stdin_cp != -1) g_StdIn.CodePage = stdin_cp; } + g_StdOut.ListPathSeparatorSlash = options.ListPathSeparatorSlash; + g_StdErr.ListPathSeparatorSlash = options.ListPathSeparatorSlash; unsigned percentsNameLevel = 1; if (options.LogLevel == 0 || options.Number_for_Percents != options.Number_for_Out) @@ -921,15 +969,16 @@ int Main2( #if !defined(UNDER_CE) CONSOLE_SCREEN_BUFFER_INFO consoleInfo; if (GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &consoleInfo)) - consoleWidth = (unsigned)(unsigned short)consoleInfo.dwSize.X; + consoleWidth = (USHORT)consoleInfo.dwSize.X; #endif #else +#if !defined(__sun) struct winsize w; if (ioctl(0, TIOCGWINSZ, &w) == 0) consoleWidth = w.ws_col; - +#endif #endif } @@ -1080,7 +1129,7 @@ int Main2( { s += " ("; s += ext.AddExt; - s += ')'; + s.Add_Char(')'); } } @@ -1103,15 +1152,15 @@ int Main2( { if (j != 0) so << ' '; - Byte b = sig[j]; + const unsigned b = sig.ConstData()[j]; if (b > 0x20 && b < 0x80) { so << (char)b; } else { - so << GetHex((b >> 4) & 0xF); - so << GetHex(b & 0xF); + so << GET_HEX_CHAR_UPPER(b >> 4); + so << GET_HEX_CHAR_UPPER(b & 15); } } } @@ -1481,6 +1530,9 @@ int Main2( } else if (options.Command.IsFromUpdateGroup()) { + #ifdef Z7_EXTRACT_ONLY + throw "update commands are not implemented"; + #else CUpdateOptions &uo = options.UpdateOptions; if (uo.SfxMode && uo.SfxModule.IsEmpty()) uo.SfxModule = kDefaultSfxModule; @@ -1536,6 +1588,7 @@ int Main2( g_StdStream, se, true // options.EnableHeaders ); + #endif } else if (options.Command.CommandType == NCommandType::kHash) { diff --git a/CPP/7zip/UI/Console/MainAr.cpp b/CPP/7zip/UI/Console/MainAr.cpp index 80f84f5..dca05a8 100644 --- a/CPP/7zip/UI/Console/MainAr.cpp +++ b/CPP/7zip/UI/Console/MainAr.cpp @@ -61,6 +61,8 @@ static inline bool CheckIsa() { // __try { + // some compilers (e2k) support SSE/AVX, but cpuid() can be unavailable or return lower isa support +#ifdef MY_CPU_X86_OR_AMD64 #if defined(__AVX2__) if (!CPU_IsSupported_AVX2()) return false; @@ -75,6 +77,7 @@ static inline bool CheckIsa() !CPU_IsSupported_CMOV()) return false; #endif +#endif /* __asm { diff --git a/CPP/7zip/UI/Console/PercentPrinter.cpp b/CPP/7zip/UI/Console/PercentPrinter.cpp index 9d392ab..1e3cfce 100644 --- a/CPP/7zip/UI/Console/PercentPrinter.cpp +++ b/CPP/7zip/UI/Console/PercentPrinter.cpp @@ -143,7 +143,7 @@ void CPercentPrinter::Print() _s.Add_Space(); _tempU = FileName; - _so->Normalize_UString(_tempU); + _so->Normalize_UString_Path(_tempU); _so->Convert_UString_to_AString(_tempU, _temp); if (_s.Len() + _temp.Len() > MaxLen) { @@ -157,7 +157,7 @@ void CPercentPrinter::Print() _tempU = FileName; _tempU.Delete(len / 2, _tempU.Len() - len); _tempU.Insert(len / 2, L" . "); - _so->Normalize_UString(_tempU); + _so->Normalize_UString_Path(_tempU); _so->Convert_UString_to_AString(_tempU, _temp); if (_s.Len() + _temp.Len() <= MaxLen) break; diff --git a/CPP/7zip/UI/Console/UpdateCallbackConsole.cpp b/CPP/7zip/UI/Console/UpdateCallbackConsole.cpp index 85496f5..3e79645 100644 --- a/CPP/7zip/UI/Console/UpdateCallbackConsole.cpp +++ b/CPP/7zip/UI/Console/UpdateCallbackConsole.cpp @@ -147,7 +147,7 @@ HRESULT CUpdateCallbackConsole::OpenResult( if (_se) { *_se << kError; - _se->NormalizePrint_wstr(name); + _se->NormalizePrint_wstr_Path(name); *_se << endl; HRESULT res = Print_OpenArchive_Error(*_se, codecs, arcLink); RINOK(res) @@ -191,7 +191,7 @@ void CCallbackConsoleBase::CommonError(const FString &path, DWORD systemError, b *_se << endl << (isWarning ? kWarning : kError) << NError::MyFormatMessage(systemError) << endl; - _se->NormalizePrint_UString(fs2us(path)); + _se->NormalizePrint_UString_Path(fs2us(path)); *_se << endl << endl; _se->Flush(); } @@ -312,7 +312,7 @@ HRESULT CUpdateCallbackConsole::StartArchive(const wchar_t *name, bool updating) { *_so << (updating ? kUpdatingArchiveMessage : kCreatingArchiveMessage); if (name) - _so->NormalizePrint_wstr(name); + _so->NormalizePrint_wstr_Path(name); else *_so << k_StdOut_ArcName; *_so << endl << endl; @@ -379,7 +379,7 @@ HRESULT CUpdateCallbackConsole::DeletingAfterArchiving(const FString &path, bool _tempA.Add_Space(); *_so << _tempA; _tempU = fs2us(path); - _so->Normalize_UString(_tempU); + _so->Normalize_UString_Path(_tempU); _so->PrintUString(_tempU, _tempA); *_so << endl; if (NeedFlush) @@ -516,7 +516,7 @@ HRESULT CCallbackConsoleBase::PrintProgress(const wchar_t *name, bool isDir, con _tempU = name; if (isDir) NWindows::NFile::NName::NormalizeDirPathPrefix(_tempU); - _so->Normalize_UString(_tempU); + _so->Normalize_UString_Path(_tempU); } _so->PrintUString(_tempU, _tempA); *_so << endl; @@ -642,7 +642,7 @@ HRESULT CUpdateCallbackConsole::ReportExtractResult(Int32 opRes, Int32 isEncrypt AString s; SetExtractErrorMessage(opRes, isEncrypted, s); *_se << s << " : " << endl; - _se->NormalizePrint_wstr(name); + _se->NormalizePrint_wstr_Path(name); *_se << endl << endl; _se->Flush(); } diff --git a/CPP/7zip/UI/Console/UserInputUtils.cpp b/CPP/7zip/UI/Console/UserInputUtils.cpp index 04d675e..2adf9df 100644 --- a/CPP/7zip/UI/Console/UserInputUtils.cpp +++ b/CPP/7zip/UI/Console/UserInputUtils.cpp @@ -49,6 +49,7 @@ NUserAnswerMode::EEnum ScanUserYesNoAllQuit(CStdOutStream *outStream) case kNoAll: return NUserAnswerMode::kNoAll; case kAutoRenameAll: return NUserAnswerMode::kAutoRenameAll; case kQuit: return NUserAnswerMode::kQuit; + default: break; } } } diff --git a/CPP/7zip/UI/Console/makefile.gcc b/CPP/7zip/UI/Console/makefile.gcc index 8a293d8..da995a7 100644 --- a/CPP/7zip/UI/Console/makefile.gcc +++ b/CPP/7zip/UI/Console/makefile.gcc @@ -38,15 +38,16 @@ endif -LOCAL_FLAGS_WIN= +LOCAL_FLAGS_SYS= ifdef IS_MINGW -LOCAL_FLAGS_WIN = \ - -DZ7_LARGE_PAGES \ - -DZ7_LONG_PATH \ +LOCAL_FLAGS_SYS = \ -DZ7_DEVICE_FILE \ +# -DZ7_LARGE_PAGES +# -DZ7_LONG_PATH + SYS_OBJS = \ $O/FileSystem.o \ $O/Registry.o \ @@ -64,7 +65,7 @@ endif LOCAL_FLAGS = \ - $(LOCAL_FLAGS_WIN) \ + $(LOCAL_FLAGS_SYS) \ $(LOCAL_FLAGS_ST) \ -DZ7_EXTERNAL_CODECS \ diff --git a/CPP/7zip/UI/Explorer/ContextMenu.cpp b/CPP/7zip/UI/Explorer/ContextMenu.cpp index 7815e13..d79bab1 100644 --- a/CPP/7zip/UI/Explorer/ContextMenu.cpp +++ b/CPP/7zip/UI/Explorer/ContextMenu.cpp @@ -295,8 +295,10 @@ static const CHashCommand g_HashCommands[] = { { CZipContextMenu::kHash_CRC32, "CRC-32", "CRC32" }, { CZipContextMenu::kHash_CRC64, "CRC-64", "CRC64" }, + { CZipContextMenu::kHash_XXH64, "XXH64", "XXH64" }, { CZipContextMenu::kHash_SHA1, "SHA-1", "SHA1" }, { CZipContextMenu::kHash_SHA256, "SHA-256", "SHA256" }, + { CZipContextMenu::kHash_BLAKE2SP, "BLAKE2sp", "BLAKE2sp" }, { CZipContextMenu::kHash_All, "*", "*" }, { CZipContextMenu::kHash_Generate_SHA256, "SHA-256 -> file.sha256", "SHA256" }, { CZipContextMenu::kHash_TestArc, "Checksum : Test", "Hash" } @@ -505,7 +507,7 @@ static const char * const kExtractExcludeExtensions = " sed sh shn shtml sln sql srt swa" " tcl tex tiff tta txt" " vb vcproj vbs" - " wav wma wv" + " mkv wav webm wma wv" " xml xsd xsl xslt" " "; @@ -593,9 +595,11 @@ Z7_COMWF_B CZipContextMenu::QueryContextMenu(HMENU hMenu, UINT indexMenu, } */ + #define MAKE_HRESULT_SUCCESS_FAC0(code) (HRESULT)(code) + if (_fileNames.Size() == 0) { - return MAKE_HRESULT(SEVERITY_SUCCESS, 0, 0); + return MAKE_HRESULT_SUCCESS_FAC0(0); // return E_INVALIDARG; } @@ -607,8 +611,8 @@ Z7_COMWF_B CZipContextMenu::QueryContextMenu(HMENU hMenu, UINT indexMenu, if ((flags & 0x000F) != CMF_NORMAL && (flags & CMF_VERBSONLY) == 0 && (flags & CMF_EXPLORE) == 0) - return MAKE_HRESULT(SEVERITY_SUCCESS, 0, currentCommandID - commandIDFirst); - // return MAKE_HRESULT(SEVERITY_SUCCESS, 0, currentCommandID); + return MAKE_HRESULT_SUCCESS_FAC0(currentCommandID - commandIDFirst); + // return MAKE_HRESULT_SUCCESS_FAC0(currentCommandID); // 19.01 : we changed from (currentCommandID) to (currentCommandID - commandIDFirst) // why it was so before? @@ -1162,7 +1166,7 @@ Z7_COMWF_B CZipContextMenu::QueryContextMenu(HMENU hMenu, UINT indexMenu, assigned in multiple menu items from different subhandlers. And we must add items to _commandMap before adding to menu. */ - return MAKE_HRESULT(SEVERITY_SUCCESS, 0, _commandMap.Size()); + return MAKE_HRESULT_SUCCESS_FAC0(_commandMap.Size()); COM_TRY_END } @@ -1332,8 +1336,10 @@ HRESULT CZipContextMenu::InvokeCommandCommon(const CCommandMapItem &cmi) case kHash_CRC32: case kHash_CRC64: + case kHash_XXH64: case kHash_SHA1: case kHash_SHA256: + case kHash_BLAKE2SP: case kHash_All: case kHash_Generate_SHA256: case kHash_TestArc: diff --git a/CPP/7zip/UI/Explorer/ContextMenu.h b/CPP/7zip/UI/Explorer/ContextMenu.h index 3f25f6a..a68ba9d 100644 --- a/CPP/7zip/UI/Explorer/ContextMenu.h +++ b/CPP/7zip/UI/Explorer/ContextMenu.h @@ -87,8 +87,10 @@ class CZipContextMenu Z7_final: kCompressToZipEmail, kHash_CRC32, kHash_CRC64, + kHash_XXH64, kHash_SHA1, kHash_SHA256, + kHash_BLAKE2SP, kHash_All, kHash_Generate_SHA256, kHash_TestArc diff --git a/CPP/7zip/UI/Explorer/DllExportsExplorer.cpp b/CPP/7zip/UI/Explorer/DllExportsExplorer.cpp index 7307b07..c92c983 100644 --- a/CPP/7zip/UI/Explorer/DllExportsExplorer.cpp +++ b/CPP/7zip/UI/Explorer/DllExportsExplorer.cpp @@ -66,6 +66,10 @@ extern LONG g_DllRefCount; LONG g_DllRefCount = 0; // Reference count of this DLL. +extern +bool g_DisableUserQuestions; +bool g_DisableUserQuestions; + // #define ODS(sz) OutputDebugStringW(L#sz) #define ODS(sz) diff --git a/CPP/7zip/UI/Explorer/Explorer.dsp b/CPP/7zip/UI/Explorer/Explorer.dsp index ff8503b..2ed491f 100644 --- a/CPP/7zip/UI/Explorer/Explorer.dsp +++ b/CPP/7zip/UI/Explorer/Explorer.dsp @@ -589,6 +589,14 @@ SOURCE=..\..\..\Windows\Synchronization.h # End Source File # Begin Source File +SOURCE=..\..\..\Windows\TimeUtils.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Windows\TimeUtils.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Windows\Window.cpp # End Source File # Begin Source File diff --git a/CPP/7zip/UI/Explorer/MyExplorerCommand.h b/CPP/7zip/UI/Explorer/MyExplorerCommand.h index 11ac247..41ac8d6 100644 --- a/CPP/7zip/UI/Explorer/MyExplorerCommand.h +++ b/CPP/7zip/UI/Explorer/MyExplorerCommand.h @@ -89,7 +89,7 @@ enum GETPROPERTYSTOREFLAGS #define PROPERTYKEY_DEFINED typedef -struct _tagpropertykey +struct { GUID fmtid; DWORD pid; diff --git a/CPP/7zip/UI/Explorer/MyMessages.cpp b/CPP/7zip/UI/Explorer/MyMessages.cpp index daff7de..c079e31 100644 --- a/CPP/7zip/UI/Explorer/MyMessages.cpp +++ b/CPP/7zip/UI/Explorer/MyMessages.cpp @@ -11,12 +11,15 @@ using namespace NWindows; +extern bool g_DisableUserQuestions; + void ShowErrorMessage(HWND window, LPCWSTR message) { - ::MessageBoxW(window, message, L"7-Zip", MB_OK | MB_ICONSTOP); + if (!g_DisableUserQuestions) + ::MessageBoxW(window, message, L"7-Zip", MB_OK | MB_ICONSTOP); } -void ShowErrorMessageHwndRes(HWND window, UINT resID) +void ShowErrorMessageHwndRes(HWND window, UInt32 resID) { UString s = LangString(resID); if (s.IsEmpty()) @@ -24,7 +27,7 @@ void ShowErrorMessageHwndRes(HWND window, UINT resID) ShowErrorMessage(window, s); } -void ShowErrorMessageRes(UINT resID) +void ShowErrorMessageRes(UInt32 resID) { ShowErrorMessageHwndRes(NULL, resID); } diff --git a/CPP/7zip/UI/Explorer/RegistryContextMenu.cpp b/CPP/7zip/UI/Explorer/RegistryContextMenu.cpp index bc03d6a..6563d89 100644 --- a/CPP/7zip/UI/Explorer/RegistryContextMenu.cpp +++ b/CPP/7zip/UI/Explorer/RegistryContextMenu.cpp @@ -48,17 +48,16 @@ static const bool k_shellex_Statuses[2][4] = }; -// can we use static RegDeleteKeyExW in _WIN64 mode? -// is it supported by Windows 2003 x64? - -/* -#ifdef _WIN64 - -#define INIT_REG_WOW - -#else -*/ +// RegDeleteKeyExW is supported starting from win2003sp1/xp-pro-x64 +// Z7_WIN32_WINNT_MIN < 0x0600 // Vista +#if !defined(Z7_WIN32_WINNT_MIN) \ + || Z7_WIN32_WINNT_MIN < 0x0502 /* < win2003 */ \ + || Z7_WIN32_WINNT_MIN == 0x0502 && !defined(_M_AMD64) +#define Z7_USE_DYN_RegDeleteKeyExW +#endif +#ifdef Z7_USE_DYN_RegDeleteKeyExW +Z7_DIAGNOSTIC_IGNORE_CAST_FUNCTION typedef // WINADVAPI LONG (APIENTRY *Func_RegDeleteKeyExW)(HKEY hKey, LPCWSTR lpSubKey, REGSAM samDesired, DWORD Reserved); @@ -71,25 +70,24 @@ static void Init_RegDeleteKeyExW() Func_RegDeleteKeyExW, GetModuleHandleW(L"advapi32.dll"), "RegDeleteKeyExW"); } - #define INIT_REG_WOW if (wow != 0) Init_RegDeleteKeyExW(); +#else +#define INIT_REG_WOW +#endif -// #endif static LONG MyRegistry_DeleteKey(HKEY parentKey, LPCTSTR name, UInt32 wow) { if (wow == 0) return RegDeleteKey(parentKey, name); - - /* - #ifdef _WIN64 - return RegDeleteKeyExW - #else - */ + +#ifdef Z7_USE_DYN_RegDeleteKeyExW if (!func_RegDeleteKeyExW) return E_NOTIMPL; return func_RegDeleteKeyExW - // #endif +#else + return RegDeleteKeyExW +#endif (parentKey, GetUnicodeString(name), wow, 0); } diff --git a/CPP/7zip/UI/Explorer/makefile b/CPP/7zip/UI/Explorer/makefile index da63a5c..3901d6b 100644 --- a/CPP/7zip/UI/Explorer/makefile +++ b/CPP/7zip/UI/Explorer/makefile @@ -7,7 +7,8 @@ CFLAGS = $(CFLAGS) \ LIBS = $(LIBS) Commctrl.lib !ELSE LIBS = $(LIBS) htmlhelp.lib comdlg32.lib Mpr.lib Gdi32.lib -CFLAGS = $(CFLAGS) -DZ7_LONG_PATH +# CFLAGS = $(CFLAGS) -DZ7_LONG_PATH +# -DZ7_NO_LARGE_PAGES !ENDIF EXPLORER_OBJS = \ @@ -41,6 +42,7 @@ WIN_OBJS = \ $O\ResourceString.obj \ $O\Shell.obj \ $O\Synchronization.obj \ + $O\TimeUtils.obj \ $O\Window.obj \ !IFDEF UNDER_CE diff --git a/CPP/7zip/UI/Far/Far.cpp b/CPP/7zip/UI/Far/Far.cpp index 74d5e60..211dde8 100644 --- a/CPP/7zip/UI/Far/Far.cpp +++ b/CPP/7zip/UI/Far/Far.cpp @@ -346,7 +346,7 @@ static HANDLE MyOpenFilePlugin(const char *name, bool isAbortCodeSupported) return MyOpenFilePluginW(GetUnicodeString(name, codePage), isAbortCodeSupported); } -EXTERN_C HANDLE WINAPI OpenFilePlugin(char *name, const unsigned char * /* data */, int /* dataSize */) +EXTERN_C HANDLE WINAPI OpenFilePlugin(char *name, const Byte * /* data */, int /* dataSize */) { MY_TRY_BEGIN // OutputDebugStringA("--- OpenFilePlugin"); @@ -360,7 +360,7 @@ EXTERN_C HANDLE WINAPI OpenFilePlugin(char *name, const unsigned char * /* data } /* -EXTERN_C HANDLE WINAPI OpenFilePluginW(const wchar_t *name,const unsigned char *Data,int DataSize,int OpMode) +EXTERN_C HANDLE WINAPI OpenFilePluginW(const wchar_t *name,const Byte *Data,int DataSize,int OpMode) { MY_TRY_BEGIN if (name == NULL || (!g_Options.Enabled)) @@ -553,7 +553,7 @@ EXTERN_C int WINAPI DeleteFiles(HANDLE plugin, PluginPanelItem *panelItems, int MY_TRY_END2("DeleteFiles", FALSE) } -EXTERN_C int WINAPI ProcessKey(HANDLE plugin, int key, unsigned int controlState) +EXTERN_C int WINAPI ProcessKey(HANDLE plugin, int key, unsigned controlState) { MY_TRY_BEGIN /* FIXME: after folder creation with F7, it doesn't reload new file list diff --git a/CPP/7zip/UI/Far/FarPlugin.h b/CPP/7zip/UI/Far/FarPlugin.h index ad3ed38..68c767b 100644 --- a/CPP/7zip/UI/Far/FarPlugin.h +++ b/CPP/7zip/UI/Far/FarPlugin.h @@ -7,7 +7,7 @@ const int kInfoPanelLineSize = 80; // #define __FAR_PLUGIN_H #ifdef UNDER_CE -typedef struct _CHAR_INFO { +typedef struct { union { WCHAR UnicodeChar; CHAR AsciiChar; @@ -29,9 +29,9 @@ typedef struct _CHAR_INFO { #endif #endif - #if _MSC_VER + // #if _MSC_VER #define _export - #endif + // #endif #define NM 260 @@ -82,7 +82,7 @@ typedef int (WINAPI *FARAPIMENU)( int X, int Y, int MaxHeight, - unsigned int Flags, + unsigned Flags, char *Title, char *Bottom, char *HelpTopic, @@ -122,7 +122,7 @@ enum { typedef int (WINAPI *FARAPIMESSAGE)( INT_PTR PluginNumber, - unsigned int Flags, + unsigned Flags, const char *HelpTopic, const char * const *Items, int ItemsNumber, @@ -177,7 +177,7 @@ struct FarDialogItem int ListPos; CHAR_INFO *VBuf; }; - unsigned int Flags; + unsigned Flags; int DefaultButton; char Data[512]; }; @@ -495,7 +495,7 @@ enum OPERATION_MODES { EXTERN_C_BEGIN void WINAPI _export ClosePluginW(HANDLE hPlugin); - int WINAPI _export CompareW(HANDLE hPlugin,const struct PluginPanelItem *Item1,const struct PluginPanelItem *Item2,unsigned int Mode); + int WINAPI _export CompareW(HANDLE hPlugin,const struct PluginPanelItem *Item1,const struct PluginPanelItem *Item2,unsigned Mode); int WINAPI _export ConfigureW(int ItemNumber); int WINAPI _export DeleteFilesW(HANDLE hPlugin,struct PluginPanelItem *PanelItem,int ItemsNumber,int OpMode); void WINAPI _export ExitFARW(void); @@ -515,7 +515,7 @@ EXTERN_C_BEGIN int WINAPI _export ProcessEditorInputW(const INPUT_RECORD *Rec); int WINAPI _export ProcessEventW(HANDLE hPlugin,int Event,void *Param); int WINAPI _export ProcessHostFileW(HANDLE hPlugin,struct PluginPanelItem *PanelItem,int ItemsNumber,int OpMode); - int WINAPI _export ProcessKeyW(HANDLE hPlugin,int Key,unsigned int ControlState); + int WINAPI _export ProcessKeyW(HANDLE hPlugin,int Key,unsigned ControlState); int WINAPI _export ProcessSynchroEventW(int Event,void *Param); int WINAPI _export ProcessViewerEventW(int Event,void *Param); int WINAPI _export PutFilesW(HANDLE hPlugin,struct PluginPanelItem *PanelItem,int ItemsNumber,int Move,const wchar_t *SrcPath,int OpMode); @@ -528,7 +528,7 @@ EXTERN_C_END EXTERN_C_BEGIN void WINAPI _export ClosePlugin(HANDLE hPlugin); - int WINAPI _export Compare(HANDLE hPlugin,const struct PluginPanelItem *Item1,const struct PluginPanelItem *Item2,unsigned int Mode); + int WINAPI _export Compare(HANDLE hPlugin,const struct PluginPanelItem *Item1,const struct PluginPanelItem *Item2,unsigned Mode); int WINAPI _export Configure(int ItemNumber); int WINAPI _export DeleteFiles(HANDLE hPlugin,struct PluginPanelItem *PanelItem,int ItemsNumber,int OpMode); void WINAPI _export ExitFAR(void); @@ -541,14 +541,14 @@ EXTERN_C_BEGIN void WINAPI _export GetPluginInfo(struct PluginInfo *Info); int WINAPI _export GetVirtualFindData(HANDLE hPlugin,struct PluginPanelItem **pPanelItem,int *pItemsNumber,const char *Path); int WINAPI _export MakeDirectory(HANDLE hPlugin,char *Name,int OpMode); - HANDLE WINAPI _export OpenFilePlugin(char *Name,const unsigned char *Data,int DataSize); + HANDLE WINAPI _export OpenFilePlugin(char *Name,const BYTE *Data,int DataSize); HANDLE WINAPI _export OpenPlugin(int OpenFrom,INT_PTR Item); int WINAPI _export ProcessDialogEvent(int Event,void *Param); int WINAPI _export ProcessEditorEvent(int Event,void *Param); int WINAPI _export ProcessEditorInput(const INPUT_RECORD *Rec); int WINAPI _export ProcessEvent(HANDLE hPlugin,int Event,void *Param); int WINAPI _export ProcessHostFile(HANDLE hPlugin,struct PluginPanelItem *PanelItem,int ItemsNumber,int OpMode); - int WINAPI _export ProcessKey(HANDLE hPlugin,int Key,unsigned int ControlState); + int WINAPI _export ProcessKey(HANDLE hPlugin,int Key,unsigned ControlState); int WINAPI _export ProcessViewerEvent(int Event,void *Param); int WINAPI _export PutFiles(HANDLE hPlugin,struct PluginPanelItem *PanelItem,int ItemsNumber,int Move,int OpMode); int WINAPI _export SetDirectory(HANDLE hPlugin,const char *Dir,int OpMode); diff --git a/CPP/7zip/UI/Far/FarUtils.cpp b/CPP/7zip/UI/Far/FarUtils.cpp index 9fddbc1..ed61ccc 100644 --- a/CPP/7zip/UI/Far/FarUtils.cpp +++ b/CPP/7zip/UI/Far/FarUtils.cpp @@ -35,7 +35,7 @@ const char *CStartupInfo::GetMsgString(int messageId) return (const char*)m_Data.GetMsg(m_Data.ModuleNumber, messageId); } -int CStartupInfo::ShowMessage(unsigned int flags, +int CStartupInfo::ShowMessage(UInt32 flags, const char *helpTopic, const char **items, unsigned numItems, int numButtons) { return m_Data.Message(m_Data.ModuleNumber, flags, helpTopic, @@ -186,8 +186,8 @@ void CStartupInfo::InitDialogItems(const CInitDialogItem *srcItems, MyStringCopy(destItem.Data, GetMsgString(srcItem.DataMessageId)); /* - if ((unsigned int)Init[i].Data < 0xFFF) - MyStringCopy(destItem.Data, GetMsg((unsigned int)srcItem.Data)); + if ((unsigned)Init[i].Data < 0xFFF) + MyStringCopy(destItem.Data, GetMsg((unsigned)srcItem.Data)); else MyStringCopy(destItem.Data,srcItem.Data); */ @@ -367,7 +367,7 @@ int CStartupInfo::Menu( int x, int y, int maxHeight, - unsigned int flags, + unsigned flags, const char *title, const char *aBottom, const char *helpTopic, @@ -384,7 +384,7 @@ int CStartupInfo::Menu( } int CStartupInfo::Menu( - unsigned int flags, + unsigned flags, const char *title, const char *helpTopic, struct FarMenuItem *items, @@ -395,7 +395,7 @@ int CStartupInfo::Menu( } int CStartupInfo::Menu( - unsigned int flags, + unsigned flags, const char *title, const char *helpTopic, const AStringVector &items, @@ -412,7 +412,7 @@ int CStartupInfo::Menu( MyStringCopy(item.Text, reducedString); farMenuItems.Add(item); } - return Menu(flags, title, helpTopic, &farMenuItems.Front(), farMenuItems.Size()); + return Menu(flags, title, helpTopic, farMenuItems.NonConstData(), farMenuItems.Size()); } diff --git a/CPP/7zip/UI/Far/FarUtils.h b/CPP/7zip/UI/Far/FarUtils.h index 6e6ff3c..fefbc22 100644 --- a/CPP/7zip/UI/Far/FarUtils.h +++ b/CPP/7zip/UI/Far/FarUtils.h @@ -36,7 +36,7 @@ struct CInitDialogItem int X1,Y1,X2,Y2; bool Focus; bool Selected; - unsigned int Flags; //FarDialogItemFlags Flags; + UInt32 Flags; //FarDialogItemFlags Flags; bool DefaultButton; int DataMessageId; const char *DataString; @@ -60,7 +60,7 @@ class CStartupInfo const char *pluginNameForRegistry); const char *GetMsgString(int messageId); - int ShowMessage(unsigned int flags, const char *helpTopic, + int ShowMessage(UInt32 flags, const char *helpTopic, const char **items, unsigned numItems, int numButtons); int ShowWarningWithOk(const char **items, unsigned numItems); @@ -112,7 +112,7 @@ class CStartupInfo int x, int y, int maxHeight, - unsigned int flags, + unsigned flags, const char *title, const char *aBottom, const char *helpTopic, @@ -121,14 +121,14 @@ class CStartupInfo FarMenuItem *items, unsigned numItems); int Menu( - unsigned int flags, + unsigned flags, const char *title, const char *helpTopic, FarMenuItem *items, unsigned numItems); int Menu( - unsigned int flags, + unsigned flags, const char *title, const char *helpTopic, const AStringVector &items, diff --git a/CPP/7zip/UI/Far/OverwriteDialogFar.cpp b/CPP/7zip/UI/Far/OverwriteDialogFar.cpp index a45d2b2..4d1b971 100644 --- a/CPP/7zip/UI/Far/OverwriteDialogFar.cpp +++ b/CPP/7zip/UI/Far/OverwriteDialogFar.cpp @@ -46,7 +46,7 @@ static void SetFileInfoStrings(const CFileInfo &fileInfo, fileInfoStrings.Time.Empty(); if (fileInfo.TimeIsDefined) { - char timeString[32]; + char timeString[64]; ConvertUtcFileTimeToString(fileInfo.Time, timeString); fileInfoStrings.Time = g_StartupInfo.GetMsgString(NMessageID::kOverwriteModifiedOn); fileInfoStrings.Time.Add_Space(); diff --git a/CPP/7zip/UI/Far/Plugin.cpp b/CPP/7zip/UI/Far/Plugin.cpp index fe6fcce..2d31b8a 100644 --- a/CPP/7zip/UI/Far/Plugin.cpp +++ b/CPP/7zip/UI/Far/Plugin.cpp @@ -509,7 +509,7 @@ void CPlugin::GetOpenPluginInfo(struct OpenPluginInfo *info) m_PannelTitle = ' '; m_PannelTitle += _archiveTypeName; - m_PannelTitle += ':'; + m_PannelTitle.Add_Colon(); m_PannelTitle += name; m_PannelTitle.Add_Space(); if (!m_CurrentDir.IsEmpty()) @@ -686,14 +686,9 @@ struct CArchiveItemProperty VARTYPE Type; }; -static inline char GetHex_Upper(unsigned v) +static inline char GetHex_A_minus10(unsigned v, unsigned a10) { - return (char)((v < 10) ? ('0' + v) : ('A' + (v - 10))); -} - -static inline char GetHex_Lower(unsigned v) -{ - return (char)((v < 10) ? ('0' + v) : ('a' + (v - 10))); + return (char)(v < 10 ? v + '0' : v + a10); } HRESULT CPlugin::ShowAttributesWindow() @@ -815,21 +810,14 @@ HRESULT CPlugin::ShowAttributesWindow() } else { - const bool needUpper = (dataSize <= 8) - && (property.ID == kpidCRC || property.ID == kpidChecksum); + const unsigned a = dataSize <= 8 + && (property.ID == kpidCRC || property.ID == kpidChecksum) + ? 'A' - 10 : 'a' - 10; for (UInt32 k = 0; k < dataSize; k++) { - unsigned b = ((const Byte *)data)[k]; - if (needUpper) - { - s += GetHex_Upper((b >> 4) & 0xF); - s += GetHex_Upper(b & 0xF); - } - else - { - s += GetHex_Lower((b >> 4) & 0xF); - s += GetHex_Lower(b & 0xF); - } + const unsigned b = ((const Byte *)data)[k]; + s += GetHex_A_minus10(b >> 4, a); + s += GetHex_A_minus10(b & 15, a); } } } @@ -866,7 +854,7 @@ HRESULT CPlugin::ShowAttributesWindow() const unsigned numDialogItems = initDialogItems.Size(); CObjArray dialogItems(numDialogItems); - g_StartupInfo.InitDialogItems(&initDialogItems.Front(), dialogItems, numDialogItems); + g_StartupInfo.InitDialogItems(initDialogItems.ConstData(), dialogItems, numDialogItems); unsigned maxLen = 0; @@ -900,7 +888,7 @@ HRESULT CPlugin::ShowAttributesWindow() return S_OK; } -int CPlugin::ProcessKey(int key, unsigned int controlState) +int CPlugin::ProcessKey(int key, unsigned controlState) { if (key == VK_F7 && controlState == 0) { diff --git a/CPP/7zip/UI/Far/Plugin.h b/CPP/7zip/UI/Far/Plugin.h index 00ccc81..84f6bed 100644 --- a/CPP/7zip/UI/Far/Plugin.h +++ b/CPP/7zip/UI/Far/Plugin.h @@ -86,7 +86,7 @@ class CPlugin HRESULT ShowAttributesWindow(); - int ProcessKey(int key, unsigned int controlState); + int ProcessKey(int key, unsigned controlState); }; HRESULT CompressFiles(const CObjectVector &pluginPanelItems); diff --git a/CPP/7zip/UI/Far/PluginWrite.cpp b/CPP/7zip/UI/Far/PluginWrite.cpp index 3f1fba8..8a00049 100644 --- a/CPP/7zip/UI/Far/PluginWrite.cpp +++ b/CPP/7zip/UI/Far/PluginWrite.cpp @@ -322,7 +322,7 @@ void CParsedPath::ParsePath(const UString &path) { case NPathType::kLocal: { - int posDiskDelimiter = path.Find(kDiskDelimiter); + const int posDiskDelimiter = path.Find(kDiskDelimiter); if (posDiskDelimiter >= 0) { curPos = posDiskDelimiter + 1; @@ -352,7 +352,8 @@ UString CParsedPath::MergePath() const FOR_VECTOR (i, PathParts) { if (i != 0) - result += kDirDelimiter; + // result += kDirDelimiter; + result.Add_PathSepar(); result += PathParts[i]; } return result; diff --git a/CPP/7zip/UI/Far/makefile b/CPP/7zip/UI/Far/makefile index f3809a8..a66f9d7 100644 --- a/CPP/7zip/UI/Far/makefile +++ b/CPP/7zip/UI/Far/makefile @@ -4,7 +4,8 @@ CFLAGS = $(CFLAGS) \ -DZ7_EXTERNAL_CODECS \ !IFNDEF UNDER_CE -CFLAGS = $(CFLAGS) -DZ7_LONG_PATH +# CFLAGS = $(CFLAGS) -DZ7_LONG_PATH +# -DZ7_NO_LARGE_PAGES !ENDIF CURRENT_OBJS = \ diff --git a/CPP/7zip/UI/FileManager/AltStreamsFolder.cpp b/CPP/7zip/UI/FileManager/AltStreamsFolder.cpp index b3d354d..446f6de 100644 --- a/CPP/7zip/UI/FileManager/AltStreamsFolder.cpp +++ b/CPP/7zip/UI/FileManager/AltStreamsFolder.cpp @@ -462,7 +462,7 @@ Z7_COM7F_IMF(CAltStreamsFolder::CreateFile(const wchar_t *name, IProgress * /* p FString absPath; GetAbsPath(name, absPath); NIO::COutFile outFile; - if (!outFile.Create(absPath, false)) + if (!outFile.Create_NEW(absPath)) return GetLastError_noZero_HRESULT(); return S_OK; } @@ -566,7 +566,7 @@ EXTERN_C_END // static Func_NtSetInformationFile f_NtSetInformationFile; // static bool g_NtSetInformationFile_WasRequested = false; - +Z7_DIAGNOSTIC_IGNORE_CAST_FUNCTION Z7_COM7F_IMF(CAltStreamsFolder::Rename(UInt32 index, const wchar_t *newName, IProgress *progress)) { diff --git a/CPP/7zip/UI/FileManager/App.cpp b/CPP/7zip/UI/FileManager/App.cpp index 3461c92..d049fc9 100644 --- a/CPP/7zip/UI/FileManager/App.cpp +++ b/CPP/7zip/UI/FileManager/App.cpp @@ -35,7 +35,6 @@ using namespace NDir; using namespace NFind; using namespace NName; -extern DWORD g_ComCtl32Version; extern HINSTANCE g_hInstance; #define kTempDirPrefix FTEXT("7zE") diff --git a/CPP/7zip/UI/FileManager/App.h b/CPP/7zip/UI/FileManager/App.h index fc54501..1e20532 100644 --- a/CPP/7zip/UI/FileManager/App.h +++ b/CPP/7zip/UI/FileManager/App.h @@ -203,11 +203,19 @@ class CApp int GetTimestampLevel() const { return Panels[LastFocusedPanel]._timestampLevel; } void SetTimestampLevel(int level) { - unsigned i; - for (i = 0; i < kNumPanelsMax; i++) + for (unsigned i = 0; i < kNumPanelsMax; i++) { CPanel &panel = Panels[i]; panel._timestampLevel = level; + } + RedrawListItems_InPanels(); + } + + void RedrawListItems_InPanels() + { + for (unsigned i = 0; i < kNumPanelsMax; i++) + { + CPanel &panel = Panels[i]; if (panel.PanelCreated) panel.RedrawListItems(); } diff --git a/CPP/7zip/UI/FileManager/BrowseDialog.cpp b/CPP/7zip/UI/FileManager/BrowseDialog.cpp index 5170302..6464ed8 100644 --- a/CPP/7zip/UI/FileManager/BrowseDialog.cpp +++ b/CPP/7zip/UI/FileManager/BrowseDialog.cpp @@ -62,6 +62,9 @@ static void MessageBox_Error_Global(HWND wnd, const wchar_t *message) #ifdef USE_MY_BROWSE_DIALOG +#if 0 +extern HINSTANCE g_hInstance; +#endif extern bool g_LVN_ITEMACTIVATE_Support; static const int kParentIndex = -1; @@ -286,6 +289,39 @@ bool CBrowseDialog::OnInit() PostMsg(Z7_WIN_WM_UPDATEUISTATE, MAKEWPARAM(Z7_WIN_UIS_CLEAR, Z7_WIN_UISF_HIDEFOCUS)); #endif +#if 0 + { + const HWND hwndTool = GetItem(IDB_BROWSE_CREATE_DIR); + if (hwndTool) + { + // Create the tooltip: + const HWND hwndTip = CreateWindowEx(0, TOOLTIPS_CLASS, NULL, + WS_POPUP | TTS_ALWAYSTIP + // | TTS_BALLOON + , CW_USEDEFAULT, CW_USEDEFAULT, CW_USEDEFAULT, CW_USEDEFAULT, + *this, NULL, g_hInstance, NULL); + if (hwndTip) + { + // Associate the tooltip with the tool: + TOOLINFOW toolInfo; + memset(&toolInfo, 0, sizeof(toolInfo)); + toolInfo.cbSize = sizeof(toolInfo); + toolInfo.hwnd = *this; + toolInfo.uFlags = TTF_IDISHWND | TTF_SUBCLASS; + toolInfo.uId = (UINT_PTR)hwndTool; + UString s; +#ifdef Z7_LANG + LangString_OnlyFromLangFile(IDM_CREATE_FOLDER, s); + s.RemoveChar(L'&'); + if (s.IsEmpty()) +#endif + s = "Create Folder"; + toolInfo.lpszText = s.Ptr_non_const(); + SendMessage(hwndTip, TTM_ADDTOOLW, 0, (LPARAM)&toolInfo); + } + } + } +#endif return CModalDialog::OnInit(); } @@ -379,7 +415,7 @@ bool CBrowseDialog::OnNotify(UINT /* controlID */, LPNMHDR header) OnItemEnter(); break; case NM_DBLCLK: - case NM_RETURN: // probabably it's unused + case NM_RETURN: // probably it's unused if (!g_LVN_ITEMACTIVATE_Support) OnItemEnter(); break; @@ -483,15 +519,19 @@ bool CBrowseDialog::GetParentPath(const UString &path, UString &parentPrefix, US int CBrowseDialog::CompareItems(LPARAM lParam1, LPARAM lParam2) const { + if (lParam1 == lParam2) return 0; if (lParam1 == kParentIndex) return -1; if (lParam2 == kParentIndex) return 1; + const CFileInfo &f1 = _files[(int)lParam1]; const CFileInfo &f2 = _files[(int)lParam2]; - const bool isDir1 = f1.IsDir(); const bool isDir2 = f2.IsDir(); - if (isDir1 && !isDir2) return -1; - if (isDir2 && !isDir1) return 1; + if (f1.IsDir()) + { + if (!isDir2) return -1; + } + else if (isDir2) return 1; int res = 0; switch (_sortIndex) @@ -508,7 +548,8 @@ static int CALLBACK CompareItems2(LPARAM lParam1, LPARAM lParam2, LPARAM lpData) return ((CBrowseDialog *)lpData)->CompareItems(lParam1, lParam2); } -static void ConvertSizeToString(UInt64 v, wchar_t *s) +wchar_t *Browse_ConvertSizeToString(UInt64 v, wchar_t *s); +wchar_t *Browse_ConvertSizeToString(UInt64 v, wchar_t *s) { char c = 0; if (v >= ((UInt64)10000 << 20)) { v >>= 30; c = 'G'; } @@ -520,8 +561,9 @@ static void ConvertSizeToString(UInt64 v, wchar_t *s) *s++ = ' '; *s++ = (wchar_t)c; *s++ = 'B'; - *s++ = 0; + *s = 0; } + return s; } // Reload changes DirPrefix. Don't send DirPrefix in pathPrefix parameter @@ -657,7 +699,7 @@ HRESULT CBrowseDialog::Reload(const UString &pathPrefix, const UString &selected if (item.iImage < 0) item.iImage = 0; _list.InsertItem(&item); - wchar_t s[32]; + wchar_t s[64]; { s[0] = 0; ConvertUtcFileTimeToString(fi.MTime, s, @@ -672,7 +714,7 @@ HRESULT CBrowseDialog::Reload(const UString &pathPrefix, const UString &selected { s[0] = 0; if (!fi.IsDir()) - ConvertSizeToString(fi.Size, s); + Browse_ConvertSizeToString(fi.Size, s); _list.SetSubItem(index, subItem++, s); } } diff --git a/CPP/7zip/UI/FileManager/BrowseDialog2.cpp b/CPP/7zip/UI/FileManager/BrowseDialog2.cpp new file mode 100644 index 0000000..59f7527 --- /dev/null +++ b/CPP/7zip/UI/FileManager/BrowseDialog2.cpp @@ -0,0 +1,1834 @@ +// BrowseDialog2.cpp + +#include "StdAfx.h" + +#ifdef UNDER_CE +#include +#endif + +#include + +#include "../../../Common/IntToString.h" +#include "../../../Common/StringConvert.h" +#include "../../../Common/Wildcard.h" + +#include "../../../Windows/DLL.h" +#include "../../../Windows/FileFind.h" +#include "../../../Windows/FileDir.h" +#include "../../../Windows/FileName.h" +#include "../../../Windows/Menu.h" +#include "../../../Windows/ProcessUtils.h" +#include "../../../Windows/PropVariantConv.h" +#include "../../../Windows/Control/ComboBox.h" +#include "../../../Windows/Control/Dialog.h" +#include "../../../Windows/Control/Edit.h" +#include "../../../Windows/Control/ListView.h" + +#include "../Explorer/MyMessages.h" + +#ifndef Z7_NO_REGISTRY +#include "HelpUtils.h" +#endif + +#include "../Common/PropIDUtils.h" + +#include "PropertyNameRes.h" +#include "RegistryUtils.h" +#include "SysIconUtils.h" +#include "FormatUtils.h" +#include "LangUtils.h" + +#include "resource.h" +#include "BrowseDialog2Res.h" +#include "BrowseDialog2.h" + +using namespace NWindows; +using namespace NFile; +using namespace NName; +using namespace NFind; + +#ifndef _UNICODE +extern bool g_IsNT; +#endif + +extern bool g_LVN_ITEMACTIVATE_Support; + +static const int kParentIndex = -1; +// static const UINT k_Message_RefreshPathEdit = WM_APP + 1; + + +static const wchar_t *k_Message_Link_operation_was_Blocked = + L"link openning was blocked by 7-Zip"; + +extern UString HResultToMessage(HRESULT errorCode); + +static void MessageBox_HResError(HWND wnd, HRESULT errorCode, const wchar_t *name) +{ + UString s = HResultToMessage(errorCode); + if (name) + { + s.Add_LF(); + s += name; + } + ShowErrorMessage(wnd, s); +} + +static void MessageBox_LastError_path(HWND wnd, const FString &path) +{ + const HRESULT hres = GetLastError_noZero_HRESULT(); + MessageBox_HResError(wnd, hres, fs2us(path)); +} + + +static const UInt32 k_EnumerateDirsLimit = 200; +static const UInt32 k_EnumerateFilesLimit = 2000; + +struct CBrowseItem +{ + unsigned MainFileIndex; + int SubFileIndex; + bool WasInterrupted; + UInt32 NumFiles; + UInt32 NumDirs; + UInt32 NumRootItems; + UInt64 Size; + + CBrowseItem(): + // MainFileIndex(0), + SubFileIndex(-1), + WasInterrupted(false), + NumFiles(0), + NumDirs(0), + NumRootItems(0), + Size(0) + {} +}; + + +struct CBrowseEnumerator +{ + FString Path; // folder path without slash at the end + CFileInfo fi; // temp + CFileInfo fi_SubFile; + CBrowseItem bi; + + void Enumerate(unsigned level); + bool NeedInterrupt() const + { + return bi.NumFiles >= k_EnumerateFilesLimit + || bi.NumDirs >= k_EnumerateDirsLimit; + } +}; + +void CBrowseEnumerator::Enumerate(unsigned level) +{ + Path.Add_PathSepar(); + const unsigned len = Path.Len(); + CObjectVector names; + { + CEnumerator enumerator; + enumerator.SetDirPrefix(Path); + while (enumerator.Next(fi)) + { + if (level == 0) + { + if (bi.NumRootItems == 0) + fi_SubFile = fi; + bi.NumRootItems++; + } + + if (fi.IsDir()) + { + bi.NumDirs++; + if (!fi.HasReparsePoint()) + names.Add(fi.Name); + } + else + { + bi.NumFiles++; + bi.Size += fi.Size; + } + + if (level != 0 || bi.NumRootItems > 1) + if (NeedInterrupt()) + { + bi.WasInterrupted = true; + return; + } + } + } + + FOR_VECTOR (i, names) + { + if (NeedInterrupt()) + { + bi.WasInterrupted = true; + return; + } + Path.DeleteFrom(len); + Path += names[i]; + Enumerate(level + 1); + } +} + + + + +class CBrowseDialog2: public NControl::CModalDialog +{ + CRecordVector _items; + CObjectVector _files; + + NControl::CListView _list; + // NControl::CEdit _pathEdit; + NControl::CComboBox _filterCombo; + + CExtToIconMap _extToIconMap; + int _sortIndex; + int _columnIndex_fileNameInDir; + int _columnIndex_NumFiles; + int _columnIndex_NumDirs; + bool _ascending; + #ifndef Z7_SFX + bool _showDots; + #endif + UString _topDirPrefix; // we don't open parent of that folder + UString DirPrefix; + + virtual bool OnInit() Z7_override; + virtual bool OnSize(WPARAM wParam, int xSize, int ySize) Z7_override; + virtual bool OnMessage(UINT message, WPARAM wParam, LPARAM lParam) Z7_override; + virtual bool OnNotify(UINT controlID, LPNMHDR header) Z7_override; + // virtual bool OnCommand(unsigned code, unsigned itemID, LPARAM lParam) Z7_override; + virtual bool OnButtonClicked(unsigned buttonID, HWND buttonHWND) Z7_override; + virtual void OnOK() Z7_override; + + bool OnKeyDown(LPNMLVKEYDOWN keyDownInfo); + + // void Post_RefreshPathEdit() { PostMsg(k_Message_RefreshPathEdit); } + bool GetParentPath(const UString &path, UString &parentPrefix, UString &name); + // Reload changes DirPrefix. Don't send DirPrefix in pathPrefix parameter + HRESULT Reload(const UString &pathPrefix, const UStringVector &selectedNames, const UString &focusedName); + HRESULT Reload(const UString &pathPrefix, const UString &selectedNames); + HRESULT Reload(); + + void ChangeSorting_and_Reload(int columnIndex); + + const CFileInfo & Get_MainFileInfo_for_realIndex(unsigned realIndex) const + { + return _files[_items[realIndex].MainFileIndex]; + } + + const FString & Get_MainFileName_for_realIndex(unsigned realIndex) const + { + return Get_MainFileInfo_for_realIndex(realIndex).Name; + } + + void Reload_WithErrorMessage(); + void OpenParentFolder(); + // void SetPathEditText(); + void PrintFileProps(UString &s, const CFileInfo &file); + void Show_FileProps_Window(const CFileInfo &file); + void OnItemEnter(); + // void FinishOnOK(); + void OnDelete(/* bool toRecycleBin */); + virtual void OnHelp() Z7_override; + bool OnContextMenu(HANDLE windowHandle, int xPos, int yPos); + + int GetRealItemIndex(int indexInListView) const + { + LPARAM param; + if (!_list.GetItemParam((unsigned)indexInListView, param)) + return (int)-1; + return (int)param; + } + + void GetSelected_RealIndexes(CUIntVector &vector); + +public: + // bool TempMode; + // bool Show_Non7zDirs_InTemp; + // int FilterIndex; // [in / out] + // CObjectVector Filters; + + UString TempFolderPath; // with slash + UString Title; + + bool IsExactTempFolder(const UString &pathPrefix) const + { + return CompareFileNames(pathPrefix, TempFolderPath) == 0; + } + + CBrowseDialog2(): + #ifndef Z7_SFX + _showDots(false) + #endif + // , TempMode(false) + // Show_Non7zDirs_InTemp(false), + // FilterIndex(-1) + {} + INT_PTR Create(HWND parent = NULL) { return CModalDialog::Create(IDD_BROWSE2, parent); } + int CompareItems(LPARAM lParam1, LPARAM lParam2) const; +}; + + +#ifdef Z7_LANG +static const UInt32 kLangIDs[] = +{ + IDS_BUTTON_DELETE, + IDM_VIEW_REFRESH +}; +#endif + +bool CBrowseDialog2::OnInit() +{ + #ifdef Z7_LANG + LangSetDlgItems(*this, kLangIDs, Z7_ARRAY_SIZE(kLangIDs)); + #endif + if (!Title.IsEmpty()) + SetText(Title); + + _list.Attach(GetItem(IDL_BROWSE2)); + _filterCombo.Attach(GetItem(IDC_BROWSE2_FILTER)); + + _ascending = true; + _sortIndex = 0; + _columnIndex_fileNameInDir = -1; + _columnIndex_NumFiles = -1; + _columnIndex_NumDirs = -1; + // _pathEdit.Attach(GetItem(IDE_BROWSE_PATH)); + + #ifndef UNDER_CE + _list.SetUnicodeFormat(); + #endif + + #ifndef Z7_SFX + { + CFmSettings st; + st.Load(); + + DWORD extendedStyle = 0; + if (st.FullRow) + extendedStyle |= LVS_EX_FULLROWSELECT; + if (st.ShowGrid) + extendedStyle |= LVS_EX_GRIDLINES; + if (st.SingleClick) + { + extendedStyle |= LVS_EX_ONECLICKACTIVATE | LVS_EX_TRACKSELECT; + /* + if (ReadUnderline()) + extendedStyle |= LVS_EX_UNDERLINEHOT; + */ + } + if (extendedStyle) + _list.SetExtendedListViewStyle(extendedStyle); + _showDots = st.ShowDots; + } + #endif + + { + /* + Filters.Clear(); // for debug + if (Filters.IsEmpty() && !FolderMode) + { + CBrowseFilterInfo &f = Filters.AddNew(); + const UString mask("*.*"); + f.Masks.Add(mask); + // f.Description = "("; + f.Description += mask; + // f.Description += ")"; + } + */ + _filterCombo.AddString(L"7-Zip temp files (7z*)"); + _filterCombo.SetCurSel(0); + EnableItem(IDC_BROWSE2_FILTER, false); +#if 0 + FOR_VECTOR (i, Filters) + { + _filterCombo.AddString(Filters[i].Description); + } + if (Filters.Size() <= 1) + { + EnableItem(IDC_BROWSE_FILTER, false); + } + if (/* FilterIndex >= 0 && */ (unsigned)FilterIndex < Filters.Size()) + _filterCombo.SetCurSel(FilterIndex); +#endif + } + + _list.SetImageList(GetSysImageList(true), LVSIL_SMALL); + _list.SetImageList(GetSysImageList(false), LVSIL_NORMAL); + + unsigned columnIndex = 0; + _list.InsertColumn(columnIndex++, LangString(IDS_PROP_NAME), 100); + _list.InsertColumn(columnIndex++, LangString(IDS_PROP_MTIME), 100); + { + LV_COLUMNW column; + column.iSubItem = (int)columnIndex; + column.mask = LVCF_FMT | LVCF_WIDTH | LVCF_TEXT | LVCF_SUBITEM; + column.fmt = LVCFMT_RIGHT; + column.cx = 100; + UString s = LangString(IDS_PROP_SIZE); + column.pszText = s.Ptr_non_const(); + _list.InsertColumn(columnIndex++, &column); + + // if (TempMode) + { + _columnIndex_NumFiles = (int)columnIndex; + s = LangString(IDS_PROP_FILES); + column.pszText = s.Ptr_non_const(); + _list.InsertColumn(columnIndex++, &column); + + _columnIndex_NumDirs = (int)columnIndex; + s = LangString(IDS_PROP_FOLDERS); + column.pszText = s.Ptr_non_const(); + _list.InsertColumn(columnIndex++, &column); + + _columnIndex_fileNameInDir = (int)columnIndex; + s = LangString(IDS_PROP_NAME); + s += "-2"; + _list.InsertColumn(columnIndex++, s, 100); + } + } + + _list.InsertItem(0, L"12345678901234567" + #ifndef UNDER_CE + L"1234567890" + #endif + ); + _list.SetSubItem(0, 1, L"2009-09-09" + #ifndef UNDER_CE + L" 09:09:09" + #endif + ); + _list.SetSubItem(0, 2, L"99999 MB+"); + + if (_columnIndex_NumFiles >= 0) + _list.SetSubItem(0, (unsigned)_columnIndex_NumFiles, L"123456789+"); + if (_columnIndex_NumDirs >= 0) + _list.SetSubItem(0, (unsigned)_columnIndex_NumDirs, L"123456789+"); + if (_columnIndex_fileNameInDir >= 0) + _list.SetSubItem(0, (unsigned)_columnIndex_fileNameInDir, L"12345678901234567890"); + + for (unsigned i = 0; i < columnIndex; i++) + _list.SetColumnWidthAuto((int)i); + _list.DeleteAllItems(); + + // if (TempMode) + { + _sortIndex = 1; // for MTime column + // _ascending = false; + } + + + NormalizeSize(); + + _topDirPrefix.Empty(); + { + unsigned rootSize = GetRootPrefixSize(TempFolderPath); + #if defined(_WIN32) && !defined(UNDER_CE) + // We can go up from root folder to drives list + if (IsDrivePath(TempFolderPath)) + rootSize = 0; + else if (IsSuperPath(TempFolderPath)) + { + if (IsDrivePath(TempFolderPath.Ptr(kSuperPathPrefixSize))) + rootSize = kSuperPathPrefixSize; + } + #endif + _topDirPrefix.SetFrom(TempFolderPath, rootSize); + } + + if (Reload(TempFolderPath, UString()) != S_OK) + { + // return false; + } +/* + UString name; + DirPrefix = TempFolderPath; + for (;;) + { + UString baseFolder = DirPrefix; + if (Reload(baseFolder, name) == S_OK) + break; + name.Empty(); + if (DirPrefix.IsEmpty()) + break; + UString parent, name2; + GetParentPath(DirPrefix, parent, name2); + DirPrefix = parent; + } +*/ + + #ifndef UNDER_CE + /* If we clear UISF_HIDEFOCUS, the focus rectangle in ListView will be visible, + even if we use mouse for pressing the button to open this dialog. */ + PostMsg(Z7_WIN_WM_UPDATEUISTATE, MAKEWPARAM(Z7_WIN_UIS_CLEAR, Z7_WIN_UISF_HIDEFOCUS)); + #endif + + /* + */ + + return CModalDialog::OnInit(); +} + + +bool CBrowseDialog2::OnSize(WPARAM /* wParam */, int xSize, int ySize) +{ + int mx, my; + { + RECT r; + GetClientRectOfItem(IDS_BUTTON_DELETE, r); + mx = r.left; + my = r.top; + } + InvalidateRect(NULL); + + const int xLim = xSize - mx; + { + RECT r; + GetClientRectOfItem(IDT_BROWSE2_FOLDER, r); + MoveItem(IDT_BROWSE2_FOLDER, r.left, r.top, xLim - r.left, RECT_SIZE_Y(r)); + } + + int bx1, bx2, by; + GetItemSizes(IDCLOSE, bx1, by); + GetItemSizes(IDHELP, bx2, by); + const int y = ySize - my - by; + const int x = xLim - bx1; + MoveItem(IDCLOSE, x - mx - bx2, y, bx1, by); + MoveItem(IDHELP, x, y, bx2, by); + /* + int yPathSize; + { + RECT r; + GetClientRectOfItem(IDE_BROWSE_PATH, r); + yPathSize = RECT_SIZE_Y(r); + _pathEdit.Move(r.left, y - my - yPathSize - my - yPathSize, xLim - r.left, yPathSize); + } + */ + // Y_Size of ComboBox is tricky. Can we use it? + int yFilterSize; + { + RECT r; + GetClientRectOfItem(IDC_BROWSE2_FILTER, r); + yFilterSize = RECT_SIZE_Y(r); + _filterCombo.Move(r.left, y - my - yFilterSize, xLim - r.left, yFilterSize); + } + { + RECT r; + GetClientRectOfItem(IDL_BROWSE2, r); + _list.Move(r.left, r.top, xLim - r.left, y - my - yFilterSize - my - r.top); + } + return false; +} + + +bool CBrowseDialog2::OnMessage(UINT message, WPARAM wParam, LPARAM lParam) +{ + /* + if (message == k_Message_RefreshPathEdit) + { + // SetPathEditText(); + return true; + } + */ + if (message == WM_CONTEXTMENU) + { + if (OnContextMenu((HANDLE)wParam, GET_X_LPARAM(lParam), GET_Y_LPARAM(lParam))) + return true; + } + return CModalDialog::OnMessage(message, wParam, lParam); +} + + +/* +bool CBrowseDialog2::OnCommand(unsigned code, unsigned itemID, LPARAM lParam) +{ + if (code == CBN_SELCHANGE) + { + switch (itemID) + { + case IDC_BROWSE2_FILTER: + { + Reload(); + return true; + } + } + } + return CModalDialog::OnCommand(code, itemID, lParam); +} +*/ + +bool CBrowseDialog2::OnNotify(UINT /* controlID */, LPNMHDR header) +{ + if (header->hwndFrom != _list) + { + if (::GetParent(header->hwndFrom) == _list) + { + // NMHDR:code is UINT + // NM_RCLICK is unsigned in windows sdk + // NM_RCLICK is int in MinGW + if (header->code == (UINT)NM_RCLICK) + { +#ifdef UNDER_CE +#define MY_NMLISTVIEW_NMITEMACTIVATE NMLISTVIEW +#else +#define MY_NMLISTVIEW_NMITEMACTIVATE NMITEMACTIVATE +#endif + MY_NMLISTVIEW_NMITEMACTIVATE *itemActivate = (MY_NMLISTVIEW_NMITEMACTIVATE *)header; + if (itemActivate->hdr.hwndFrom == HWND(_list)) + return false; + /* + POINT point; + ::GetCursorPos(&point); + ShowColumnsContextMenu(point.x, point.y); + */ + // we want to disable menu for columns. + // to return the value from a dialog procedure we must + // call SetMsgResult(val) and return true; + // NM_RCLICK : Return nonzero to not allow the default processing + SetMsgResult(TRUE); // do not allow default processing + return true; + } + } + return false; + } + + switch (header->code) + { + case LVN_ITEMACTIVATE: + if (g_LVN_ITEMACTIVATE_Support) + OnItemEnter(); + break; + case NM_DBLCLK: + case NM_RETURN: // probably it's unused + if (!g_LVN_ITEMACTIVATE_Support) + OnItemEnter(); + break; + case LVN_COLUMNCLICK: + { + const int index = LPNMLISTVIEW(header)->iSubItem; + ChangeSorting_and_Reload(index); + return false; + } + case LVN_KEYDOWN: + { + bool boolResult = OnKeyDown(LPNMLVKEYDOWN(header)); + // Post_RefreshPathEdit(); + return boolResult; + } + /* + case NM_RCLICK: + case NM_CLICK: + case LVN_BEGINDRAG: + Post_RefreshPathEdit(); + break; + */ + } + + return false; +} + +bool CBrowseDialog2::OnKeyDown(LPNMLVKEYDOWN keyDownInfo) +{ + const bool ctrl = IsKeyDown(VK_CONTROL); + // const bool alt = IsKeyDown(VK_MENU); + // const bool leftCtrl = IsKeyDown(VK_LCONTROL); + // const bool rightCtrl = IsKeyDown(VK_RCONTROL); + // const bool shift = IsKeyDown(VK_SHIFT); + + switch (keyDownInfo->wVKey) + { + case VK_BACK: + OpenParentFolder(); + return true; + case 'R': + if (ctrl) + { + Reload_WithErrorMessage(); + return true; + } + return false; + case VK_F3: + case VK_F5: + case VK_F6: + if (ctrl) + { + int index = 0; // name + if (keyDownInfo->wVKey == VK_F5) index = 1; // MTime + else if (keyDownInfo->wVKey == VK_F6) index = 2; // Size + ChangeSorting_and_Reload(index); + Reload_WithErrorMessage(); + return true; + } + return false; + case 'A': + if (ctrl) + { + // if (TempMode) + _list.SelectAll(); + return true; + } + return false; + + case VK_DELETE: + // if (TempMode) + OnDelete(/* !shift */); + return true; +#if 0 + case VK_NEXT: + case VK_PRIOR: + { + if (ctrl && !alt && !shift) + { + if (keyDownInfo->wVKey == VK_NEXT) + OnItemEnter(); + else + OpenParentFolder(); + SetMsgResult(TRUE); // to disable processing + return true; + } + break; + } +#endif + } + return false; +} + + +bool CBrowseDialog2::OnButtonClicked(unsigned buttonID, HWND buttonHWND) +{ + switch (buttonID) + { + case IDB_BROWSE2_PARENT: OpenParentFolder(); break; + case IDS_BUTTON_DELETE: + { + OnDelete(/* !IsKeyDown(VK_SHIFT) */); + break; + } + case IDM_VIEW_REFRESH: + { + Reload_WithErrorMessage(); + break; + } + default: return CModalDialog::OnButtonClicked(buttonID, buttonHWND); + } + _list.SetFocus(); + return true; +} + + + +static void PrintPropsPrefix(UString &s, UInt32 id) +{ + s.Add_LF(); + s += " "; + AddLangString(s, id); + s += ": "; +} + +wchar_t *Browse_ConvertSizeToString(UInt64 v, wchar_t *s); + +static void Browse_ConvertSizeToString(const CBrowseItem &bi, wchar_t *s) +{ + s = Browse_ConvertSizeToString(bi.Size, s); + if (bi.WasInterrupted) + { + *s++ = '+'; + *s = 0; + } +} + +void AddSizeValue(UString &s, UInt64 value); + +static void PrintProps_Size(UString &s, UInt64 size) +{ + PrintPropsPrefix(s, IDS_PROP_SIZE); +#if 1 + AddSizeValue(s, size); +#else + s.Add_UInt64(size); + if (size >= 10000) + { + s += " ("; + wchar_t temp[64]; + Browse_ConvertSizeToString(size, temp); + s += temp; + s.Add_Char(')'); + } +#endif +} + +static void PrintProps_MTime(UString &s, const CFileInfo &fi) +{ + PrintPropsPrefix(s, IDS_PROP_MTIME); + char t[64]; + ConvertUtcFileTimeToString(fi.MTime, t); + s += t; +} + + +static void PrintProps_Name(UString &s, const CFileInfo &fi) +{ + s += fs2us(fi.Name); + if (fi.IsDir()) + s.Add_PathSepar(); +} + +static void PrintProps_Attrib(UString &s, const CFileInfo &fi) +{ + PrintPropsPrefix(s, IDS_PROP_ATTRIBUTES); + char props[64]; + ConvertWinAttribToString(props, fi.Attrib); + s += props; +#if 0 + if (fi.HasReparsePoint()) + { + s.Add_LF(); + s += "IsLink: +"; + } +#endif +} + +static void PrintProps(UString &s, const CBrowseItem &bi, + const CFileInfo &fi, const CFileInfo *fi2) +{ + PrintProps_Name(s, fi); + PrintProps_Attrib(s, fi); + if (bi.NumDirs != 0) + { + PrintPropsPrefix(s, IDS_PROP_FOLDERS); + s.Add_UInt32(bi.NumDirs); + if (bi.WasInterrupted) + s += "+"; + } + if (bi.NumFiles != 0) + { + PrintPropsPrefix(s, IDS_PROP_FILES); + s.Add_UInt32(bi.NumFiles); + if (bi.WasInterrupted) + s += "+"; + } + { + PrintProps_Size(s, bi.Size); + if (bi.WasInterrupted) + s += "+"; + } + + PrintProps_MTime(s, fi); + + if (fi2) + { + s.Add_LF(); + s += "----------------"; + s.Add_LF(); + PrintProps_Name(s, *fi2); + PrintProps_Attrib(s, *fi2); + if (!fi2->IsDir()) + PrintProps_Size(s, fi2->Size); + PrintProps_MTime(s, *fi2); + } +} + + +void CBrowseDialog2::GetSelected_RealIndexes(CUIntVector &vector) +{ + vector.Clear(); + int index = -1; + for (;;) + { + index = _list.GetNextSelectedItem(index); + if (index < 0) + break; + const int realIndex = GetRealItemIndex(index); + if (realIndex >= 0) + vector.Add((unsigned)realIndex); + } +} + + +void CBrowseDialog2::PrintFileProps(UString &s, const CFileInfo &file) +{ + CFileInfo file2; + FString path = us2fs(DirPrefix); + path += file.Name; + if (!file2.Find(path)) + { + MessageBox_LastError_path(*this, path); + Reload_WithErrorMessage(); + return; + } + CBrowseEnumerator enumer; + enumer.bi.Size = file2.Size; + if (file2.IsDir() && !file2.HasReparsePoint()) + { + enumer.Path = path; + enumer.Enumerate(0); // level + } + PrintProps(s, enumer.bi, file2, + enumer.bi.NumRootItems == 1 ? &enumer.fi_SubFile : NULL); +} + + +void CBrowseDialog2::Show_FileProps_Window(const CFileInfo &file) +{ + UString s; + PrintFileProps(s, file); + MessageBoxW(*this, s, LangString(IDS_PROPERTIES), MB_OK); +} + + +void CBrowseDialog2::OnDelete(/* bool toRecycleBin */) +{ +#if 1 + // we don't want deleting in non temp folders + if (!DirPrefix.IsPrefixedBy(TempFolderPath)) + return; +#endif + + CUIntVector indices; + GetSelected_RealIndexes(indices); + if (indices.Size() == 0) + return; + { + UInt32 titleID, messageID; + UString messageParam; + UString s2; + if (indices.Size() == 1) + { + const unsigned index = indices[0]; + const CBrowseItem &bi = _items[index]; + const CFileInfo &file = _files[bi.MainFileIndex]; + PrintFileProps(s2, file); + messageParam = fs2us(file.Name); + if (file.IsDir()) + { + titleID = IDS_CONFIRM_FOLDER_DELETE; + messageID = IDS_WANT_TO_DELETE_FOLDER; + } + else + { + titleID = IDS_CONFIRM_FILE_DELETE; + messageID = IDS_WANT_TO_DELETE_FILE; + } + } + else + { + titleID = IDS_CONFIRM_ITEMS_DELETE; + messageID = IDS_WANT_TO_DELETE_ITEMS; + messageParam = NumberToString(indices.Size()); + + for (UInt32 i = 0; i < indices.Size(); i++) + { + if (i >= 10) + { + s2 += "..."; + break; + } + const CBrowseItem &bi = _items[indices[i]]; + const CFileInfo &fi = _files[bi.MainFileIndex]; + PrintProps_Name(s2, fi); + s2.Add_LF(); + } + } + UString s = MyFormatNew(messageID, messageParam); + if (!s2.IsEmpty()) + { + s.Add_LF(); + s.Add_LF(); + s += s2; + } + if (::MessageBoxW((HWND)*this, s, LangString(titleID), MB_OKCANCEL | MB_ICONQUESTION) != IDOK) + return; + } + + for (UInt32 i = 0; i < indices.Size(); i++) + { + const unsigned index = indices[i]; + bool result = true; + const CBrowseItem &bi = _items[index]; + const CFileInfo &fi = _files[bi.MainFileIndex]; + if (fi.Name.IsEmpty()) + return; // some error + const FString fullPath = us2fs(DirPrefix) + fi.Name; + if (fi.IsDir()) + result = NFile::NDir::RemoveDirWithSubItems(fullPath); + else + result = NFile::NDir::DeleteFileAlways(fullPath); + if (!result) + { + MessageBox_LastError_path(*this, fullPath); + return; + } + } + + Reload_WithErrorMessage(); +} + + +#ifndef Z7_NO_REGISTRY +#define kHelpTopic "fm/temp.htm" +void CBrowseDialog2::OnHelp() +{ + ShowHelpWindow(kHelpTopic); + CModalDialog::OnHelp(); +} +#endif + + +HRESULT StartApplication(const UString &dir, const UString &path, HWND window, CProcess &process); +HRESULT StartApplication(const UString &dir, const UString &path, HWND window, CProcess &process) +{ + UString path2 = path; + + #ifdef _WIN32 + { + const int dot = path2.ReverseFind_Dot(); + const int separ = path2.ReverseFind_PathSepar(); + if (dot < 0 || dot < separ) + path2.Add_Dot(); + } + #endif + + UINT32 result; + +#ifndef _UNICODE + if (g_IsNT) + { + SHELLEXECUTEINFOW execInfo; + execInfo.cbSize = sizeof(execInfo); + execInfo.fMask = SEE_MASK_NOCLOSEPROCESS | SEE_MASK_FLAG_DDEWAIT; + execInfo.hwnd = NULL; + execInfo.lpVerb = NULL; + execInfo.lpFile = path2; + execInfo.lpParameters = NULL; + execInfo.lpDirectory = dir.IsEmpty() ? NULL : (LPCWSTR)dir; + execInfo.nShow = SW_SHOWNORMAL; + execInfo.hProcess = NULL; +typedef BOOL (WINAPI * Func_ShellExecuteExW)(LPSHELLEXECUTEINFOW lpExecInfo); +Z7_DIAGNOSTIC_IGNORE_CAST_FUNCTION + const + Func_ShellExecuteExW + f_ShellExecuteExW = Z7_GET_PROC_ADDRESS( + Func_ShellExecuteExW, ::GetModuleHandleW(L"shell32.dll"), + "ShellExecuteExW"); + if (!f_ShellExecuteExW) + return 0; + f_ShellExecuteExW(&execInfo); + result = (UINT32)(UINT_PTR)execInfo.hInstApp; + process.Attach(execInfo.hProcess); + } + else +#endif + { + SHELLEXECUTEINFO execInfo; + execInfo.cbSize = sizeof(execInfo); + execInfo.fMask = SEE_MASK_NOCLOSEPROCESS + #ifndef UNDER_CE + | SEE_MASK_FLAG_DDEWAIT + #endif + ; + execInfo.hwnd = NULL; + execInfo.lpVerb = NULL; + const CSysString sysPath (GetSystemString(path2)); + const CSysString sysDir (GetSystemString(dir)); + execInfo.lpFile = sysPath; + execInfo.lpParameters = NULL; + execInfo.lpDirectory = + #ifdef UNDER_CE + NULL + #else + sysDir.IsEmpty() ? NULL : (LPCTSTR)sysDir + #endif + ; + execInfo.nShow = SW_SHOWNORMAL; + execInfo.hProcess = NULL; + ::ShellExecuteEx(&execInfo); + result = (UINT32)(UINT_PTR)execInfo.hInstApp; + process.Attach(execInfo.hProcess); + } + + // DEBUG_PRINT_NUM("-- ShellExecuteEx -- execInfo.hInstApp = ", result) + + if (result <= 32) + { + switch (result) + { + case SE_ERR_NOASSOC: + MessageBox_HResError(window, + GetLastError_noZero_HRESULT(), + NULL + // L"There is no application associated with the given file name extension", + ); + } + + return E_FAIL; // fixed in 15.13. Can we use it for any Windows version? + } + + return S_OK; +} + +void StartApplicationDontWait(const UString &dir, const UString &path, HWND window); +void StartApplicationDontWait(const UString &dir, const UString &path, HWND window) +{ + CProcess process; + StartApplication(dir, path, window, process); +} + + +static UString GetQuotedString2(const UString &s) +{ + UString s2 ('\"'); + s2 += s; + s2.Add_Char('\"'); + return s2; +} + + +bool CBrowseDialog2::OnContextMenu(HANDLE windowHandle, int xPos, int yPos) +{ + if (windowHandle != _list) + return false; + + CUIntVector indices; + GetSelected_RealIndexes(indices); + + // negative x,y are possible for multi-screen modes. + // x=-1 && y=-1 for keyboard call (SHIFT+F10 and others). +#if 1 // 0 : for debug + if (xPos == -1 && yPos == -1) +#endif + { +/* + if (indices.Size() == 0) + { + xPos = 0; + yPos = 0; + } + else +*/ + { + const int itemIndex = _list.GetFocusedItem(); + if (itemIndex == -1) + return false; + RECT rect; + if (!_list.GetItemRect(itemIndex, &rect, LVIR_ICON)) + return false; + // rect : rect of file icon relative to listVeiw. + xPos = (rect.left + rect.right) / 2; + yPos = (rect.top + rect.bottom) / 2; + RECT r; + GetClientRectOfItem(IDL_BROWSE2, r); + if (yPos < 0 || yPos >= RECT_SIZE_Y(r)) + yPos = 0; + } + POINT point = {xPos, yPos}; + _list.ClientToScreen(&point); + xPos = point.x; + yPos = point.y; + } + + const UInt32 k_CmdId_Delete = 1; + const UInt32 k_CmdId_Open_Explorer = 2; + const UInt32 k_CmdId_Open_7zip = 3; + const UInt32 k_CmdId_Props = 4; + int menuResult; + { + CMenu menu; + CMenuDestroyer menuDestroyer(menu); + menu.CreatePopup(); + + unsigned numMenuItems = 0; + // unsigned defaultCmd = 0; + + for (unsigned cmd = k_CmdId_Delete; cmd <= k_CmdId_Props; cmd++) + { + if (cmd == k_CmdId_Delete) + { + if (/* !TempMode || */ indices.Size() == 0) + continue; + // defaultCmd = cmd; + } + else if (indices.Size() > 1) + break; + + + if (numMenuItems != 0) + { + if (cmd == k_CmdId_Open_Explorer) + menu.AppendItem(MF_SEPARATOR, 0, (LPCTSTR)NULL); + if (cmd == k_CmdId_Props) + menu.AppendItem(MF_SEPARATOR, 0, (LPCTSTR)NULL); + } + + const UINT flags = MF_STRING; + UString s; + if (cmd == k_CmdId_Delete) + { + s = LangString(IDS_BUTTON_DELETE); + s += "\tDelete"; + } + else if (cmd == k_CmdId_Open_Explorer) + { + s = LangString(IDM_OPEN_OUTSIDE); + if (s.IsEmpty()) + s = "Open Outside"; + s += "\tShift+Enter"; + } + else if (cmd == k_CmdId_Open_7zip) + { + s = LangString(IDM_OPEN_OUTSIDE); + if (s.IsEmpty()) + s = "Open Outside"; + s += " : 7-Zip"; + } + else if (cmd == k_CmdId_Props) + { + s = LangString(IDS_PROPERTIES); + if (s.IsEmpty()) + s = "Properties"; + s += "\tAlt+Enter"; + } + else + break; + s.RemoveChar(L'&'); + menu.AppendItem(flags, cmd, s); + numMenuItems++; + } + // default item is useless for us + /* + if (defaultCmd != 0) + SetMenuDefaultItem(menu, (unsigned)defaultCmd, + FALSE); // byPos + */ + /* hwnd for TrackPopupMenuEx(): DOCS: + A handle to the window that owns the shortcut menu. + This window receives all messages from the menu. + The window does not receive a WM_COMMAND message from the menu + until the function returns. + If you specify TPM_NONOTIFY in the fuFlags parameter, + the function does not send messages to the window identified by hwnd. + */ + if (numMenuItems == 0) + return true; + menuResult = menu.Track(TPM_LEFTALIGN | TPM_RETURNCMD | TPM_NONOTIFY, + xPos, yPos, *this); + /* menu.Track() return value is zero, if the user cancels + the menu without making a selection, or if an error occurs */ + if (menuResult <= 0) + return true; + } + + if (menuResult == k_CmdId_Delete) + { + OnDelete(/* !IsKeyDown(VK_SHIFT) */); + return true; + } + + if (indices.Size() <= 1) + { + UString fullPath = DirPrefix; + if (indices.Size() != 0) + { + const CBrowseItem &bi = _items[indices[0]]; + const CFileInfo &file = _files[bi.MainFileIndex]; + if (file.HasReparsePoint()) + { + // we don't want external program was used to work with Links + ShowErrorMessage(*this, k_Message_Link_operation_was_Blocked); + return true; + } + fullPath += fs2us(file.Name); + } + if (menuResult == k_CmdId_Open_Explorer) + { + StartApplicationDontWait(DirPrefix, fullPath, (HWND)*this); + return true; + } + + if (menuResult == k_CmdId_Open_7zip) + { + UString imageName = fs2us(NWindows::NDLL::GetModuleDirPrefix()); + imageName += "7zFM.exe"; + WRes wres; + { + CProcess process; + wres = process.Create(imageName, GetQuotedString2(fullPath), NULL); // curDir + } + if (wres != 0) + { + const HRESULT hres = HRESULT_FROM_WIN32(wres); + MessageBox_HResError(*this, hres, imageName); + } + return true; + } + + if (indices.Size() == 1) + if (menuResult == k_CmdId_Props) + { + const CBrowseItem &bi = _items[indices[0]]; + const CFileInfo &file = _files[bi.MainFileIndex]; + Show_FileProps_Window(file); + return true; + } + } + + return true; +} + + + +struct CWaitCursor2 +{ + HCURSOR _waitCursor; + HCURSOR _oldCursor; + + CWaitCursor2(): + _waitCursor(NULL), + _oldCursor(NULL) + {} + void Set() + { + if (!_waitCursor) + { + _waitCursor = LoadCursor(NULL, IDC_WAIT); + if (_waitCursor) + _oldCursor = SetCursor(_waitCursor); + } + } + ~CWaitCursor2() + { + if (_waitCursor) + SetCursor(_oldCursor); + } +}; + + +void CBrowseDialog2::OnOK() +{ + /* DOCS: + If a dialog box or one of its controls currently has the input focus, + then pressing the ENTER key causes Windows to send a WM_COMMAND message + with the idItem (wParam) parameter set to the ID of the default command button. + If the dialog box does not have a default command button, + then the idItem parameter is set to IDOK by default. + + We process IDOK here for Enter pressing, because we have no DEFPUSHBUTTON. + */ + if (GetFocus() == _list) + { + OnItemEnter(); + return; + } + // Enter can be pressed in another controls (Edit). + // So we don't need End() call here +} + + +bool CBrowseDialog2::GetParentPath(const UString &path, UString &parentPrefix, UString &name) +{ + parentPrefix.Empty(); + name.Empty(); + if (path.IsEmpty()) + return false; + if (_topDirPrefix == path) + return false; + UString s = path; + if (IS_PATH_SEPAR(s.Back())) + s.DeleteBack(); + if (s.IsEmpty()) + return false; + if (IS_PATH_SEPAR(s.Back())) + return false; + const unsigned pos1 = (unsigned)(s.ReverseFind_PathSepar() + 1); + parentPrefix.SetFrom(s, pos1); + name = s.Ptr(pos1); + return true; +} + + +int CBrowseDialog2::CompareItems(LPARAM lParam1, LPARAM lParam2) const +{ + if (lParam1 == lParam2) return 0; + if (lParam1 == kParentIndex) return -1; + if (lParam2 == kParentIndex) return 1; + + const int index1 = (int)lParam1; + const int index2 = (int)lParam2; + + const CBrowseItem &item1 = _items[index1]; + const CBrowseItem &item2 = _items[index2]; + + const CFileInfo &f1 = _files[item1.MainFileIndex]; + const CFileInfo &f2 = _files[item2.MainFileIndex]; + + const bool isDir2 = f2.IsDir(); + if (f1.IsDir()) + { + if (!isDir2) return -1; + } + else if (isDir2) return 1; + + const int res2 = MyCompare(index1, index2); + int res = 0; + switch (_sortIndex) + { + case 0: res = CompareFileNames(fs2us(f1.Name), fs2us(f2.Name)); break; + case 1: res = CompareFileTime(&f1.MTime, &f2.MTime); break; + case 2: res = MyCompare(item1.Size, item2.Size); break; + case 3: res = MyCompare(item1.NumFiles, item2.NumFiles); break; + case 4: res = MyCompare(item1.NumDirs, item2.NumDirs); break; + case 5: + { + const int sub1 = item1.SubFileIndex; + const int sub2 = item2.SubFileIndex; + if (sub1 < 0) + { + if (sub2 >= 0) + res = -1; + } + else if (sub2 < 0) + res = 1; + else + res = CompareFileNames(fs2us(_files[sub1].Name), fs2us(_files[sub2].Name)); + break; + } + } + if (res == 0) + res = res2; + return _ascending ? res: -res; +} + +static int CALLBACK CompareItems2(LPARAM lParam1, LPARAM lParam2, LPARAM lpData) +{ + return ((CBrowseDialog2 *)lpData)->CompareItems(lParam1, lParam2); +} + + +static const FChar *FindNonHexChar_F(const FChar *s) throw() +{ + for (;;) + { + const FChar c = (FChar)*s++; // pointer can go 1 byte after end + if ( (c < '0' || c > '9') + && (c < 'a' || c > 'z') + && (c < 'A' || c > 'Z')) + return s - 1; + } +} + + +void CBrowseDialog2::Reload_WithErrorMessage() +{ + const HRESULT res = Reload(); + if (res != S_OK) + MessageBox_HResError(*this, res, DirPrefix); +} + +void CBrowseDialog2::ChangeSorting_and_Reload(int columnIndex) +{ + if (columnIndex == _sortIndex) + _ascending = !_ascending; + else + { + _ascending = (columnIndex == 0 || columnIndex == _columnIndex_fileNameInDir); // for name columns + _sortIndex = columnIndex; + } + Reload_WithErrorMessage(); +} + + +// Reload changes DirPrefix. Don't send DirPrefix in pathPrefix parameter +HRESULT CBrowseDialog2::Reload(const UString &pathPrefix, const UString &selectedName) +{ + UStringVector selectedVector; + if (!selectedName.IsEmpty()) + selectedVector.Add(selectedName); + return Reload(pathPrefix, selectedVector, selectedName); +} + + +HRESULT CBrowseDialog2::Reload(const UString &pathPrefix, const UStringVector &selectedVector2, const UString &focusedName) +{ + UStringVector selectedVector = selectedVector2; + selectedVector.Sort(); + CObjectVector files; + CRecordVector items; + CWaitCursor2 waitCursor; + + #ifndef UNDER_CE + bool isDrive = false; + if (pathPrefix.IsEmpty() || pathPrefix.IsEqualTo(kSuperPathPrefix)) + { + isDrive = true; + FStringVector drives; + if (!MyGetLogicalDriveStrings(drives)) + return GetLastError_noZero_HRESULT(); + FOR_VECTOR (i, drives) + { + const FString &d = drives[i]; + if (d.Len() < 2 || d.Back() != '\\') + return E_FAIL; + CBrowseItem item; + item.MainFileIndex = files.Size(); + CFileInfo &fi = files.AddNew(); + fi.SetAsDir(); + fi.Name = d; + fi.Name.DeleteBack(); + items.Add(item); + } + } + else + #endif + { + { + CEnumerator enumerator; + enumerator.SetDirPrefix(us2fs(pathPrefix)); + CFileInfo fi; + FString tail; + + const bool isTempFolder = ( + // TempMode && + IsExactTempFolder(pathPrefix) + ); + for (;;) + { + { + bool found; + if (!enumerator.Next(fi, found)) + return GetLastError_noZero_HRESULT(); + if (!found) + break; + } + if (isTempFolder) + { + // if (!Show_Non7zDirs_InTemp) + { + if (!fi.Name.IsPrefixedBy_Ascii_NoCase("7z")) + continue; + tail = fi.Name.Ptr(2); + if ( !tail.IsPrefixedBy_Ascii_NoCase("E") // drag and drop / Copy / create to email + && !tail.IsPrefixedBy_Ascii_NoCase("O") // open + && !tail.IsPrefixedBy_Ascii_NoCase("S")) // SFXSetup + continue; + const FChar *beg = tail.Ptr(1); + const FChar *end = FindNonHexChar_F(beg); + if (end - beg != 8 || *end != 0) + continue; + } + } + CBrowseItem item; + item.MainFileIndex = files.Size(); + item.Size = fi.Size; + files.Add(fi); + items.Add(item); + } + } + + UInt64 cnt = items.Size(); + // if (TempMode) + { + FOR_VECTOR (i, items) + { + CBrowseItem &item = items[i]; + const CFileInfo &fi = files[item.MainFileIndex]; + if (!fi.IsDir() || fi.HasReparsePoint()) + continue; + + CBrowseEnumerator enumer; + // we need to keep MainFileIndex and Size value of item: + enumer.bi = item; // don't change it + enumer.Path = us2fs(pathPrefix); + enumer.Path += fi.Name; + enumer.Enumerate(0); // level + item = enumer.bi; + if (item.NumRootItems == 1) + { + item.SubFileIndex = (int)files.Size(); + files.Add(enumer.fi_SubFile); + } + cnt += item.NumDirs; + cnt += item.NumFiles; + if (cnt > 1000) + waitCursor.Set(); + } + } + } + _items = items; + _files = files; + + DirPrefix = pathPrefix; + + EnableItem(IDB_BROWSE2_PARENT, !IsExactTempFolder(pathPrefix)); + + SetItemText(IDT_BROWSE2_FOLDER, DirPrefix); + + _list.SetRedraw(false); + _list.DeleteAllItems(); + + LVITEMW item; + + unsigned index = 0; + int cursorIndex = -1; + + #ifndef Z7_SFX + if (_showDots && _topDirPrefix != DirPrefix) + { + item.iItem = (int)index; + const UString itemName (".."); + if (focusedName == itemName) + cursorIndex = (int)index; + /* + if (selectedVector.IsEmpty() + // && focusedName.IsEmpty() + // && focusedName == ".." + ) + cursorIndex = (int)index; + */ + item.mask = LVIF_TEXT | LVIF_PARAM | LVIF_IMAGE; + unsigned subItem = 0; + item.iSubItem = (int)(subItem++); + item.lParam = kParentIndex; + item.pszText = itemName.Ptr_non_const(); + item.iImage = _extToIconMap.GetIconIndex(FILE_ATTRIBUTE_DIRECTORY, DirPrefix); + if (item.iImage < 0) + item.iImage = 0; + _list.InsertItem(&item); +#if 0 + for (int k = 1; k < 6; k++) + _list.SetSubItem(index, subItem++, L"2"); +#endif + index++; + } + #endif + + for (unsigned i = 0; i < _items.Size(); i++, index++) + { + item.iItem = (int)index; + const CBrowseItem &bi = _items[i]; + const CFileInfo &fi = _files[bi.MainFileIndex]; + const UString name = fs2us(fi.Name); + // if (!selectedName.IsEmpty() && CompareFileNames(name, selectedName) == 0) + item.mask = LVIF_TEXT | LVIF_PARAM | LVIF_IMAGE; + item.state = 0; + if (selectedVector.FindInSorted(name) != -1) + { + /* + if (cursorIndex == -1) + cursorIndex = (int)index; + */ + item.mask |= LVIF_STATE; + item.state |= LVIS_SELECTED; + } + if (focusedName == name) + { + if (cursorIndex == -1) + cursorIndex = (int)index; + item.mask |= LVIF_STATE; + item.state |= LVIS_FOCUSED; + } + + unsigned subItem = 0; + item.iSubItem = (int)(subItem++); + item.lParam = (LPARAM)i; + item.pszText = name.Ptr_non_const(); + + const UString fullPath = DirPrefix + name; + #ifndef UNDER_CE + if (isDrive) + { + if (GetRealIconIndex(fi.Name + FCHAR_PATH_SEPARATOR, FILE_ATTRIBUTE_DIRECTORY, item.iImage) == 0) + item.iImage = 0; + } + else + #endif + item.iImage = _extToIconMap.GetIconIndex(fi.Attrib, fullPath); + if (item.iImage < 0) + item.iImage = 0; + + _list.InsertItem(&item); + wchar_t s[64]; + { + s[0] = 0; + if (!FILETIME_IsZero(fi.MTime)) + ConvertUtcFileTimeToString(fi.MTime, s, + #ifndef UNDER_CE + kTimestampPrintLevel_MIN + #else + kTimestampPrintLevel_DAY + #endif + ); + _list.SetSubItem(index, subItem++, s); + } + + { + s[0] = 0; + Browse_ConvertSizeToString(bi, s); + _list.SetSubItem(index, subItem++, s); + } + if (_columnIndex_NumFiles >= 0) + { + UString s2; + if (fi.HasReparsePoint()) + { + s2 = "Link"; + } + else if (bi.NumFiles != 0) + { + s2.Add_UInt32(bi.NumFiles); + if (bi.WasInterrupted) + s2 += "+"; + } + _list.SetSubItem(index, subItem, s2); + } + subItem++; + if (_columnIndex_NumDirs >= 0 && bi.NumDirs != 0) + { + UString s2; + s2.Add_UInt32(bi.NumDirs); + if (bi.WasInterrupted) + s2 += "+"; + _list.SetSubItem(index, subItem, s2); + } + subItem++; + if (_columnIndex_fileNameInDir >= 0 && bi.SubFileIndex >= 0) + { + _list.SetSubItem(index, subItem, fs2us(_files[bi.SubFileIndex].Name)); + } + subItem++; + } + + if (_list.GetItemCount() > 0 && cursorIndex >= 0) + { + // _list.SetItemState_FocusedSelected(cursorIndex); + // _list.SetItemState_Focused(cursorIndex); + } + _list.SortItems(CompareItems2, (LPARAM)this); + if (_list.GetItemCount() > 0 && cursorIndex < 0) + { + if (selectedVector.IsEmpty()) + _list.SetItemState_FocusedSelected(0); + else + _list.SetItemState(0, LVIS_FOCUSED, LVIS_FOCUSED); + } + _list.EnsureVisible(_list.GetFocusedItem(), false); + _list.SetRedraw(true); + _list.InvalidateRect(NULL, true); + return S_OK; +} + + + +HRESULT CBrowseDialog2::Reload() +{ + UStringVector selected; + { + CUIntVector indexes; + GetSelected_RealIndexes(indexes); + FOR_VECTOR (i, indexes) + selected.Add(fs2us(Get_MainFileName_for_realIndex(indexes[i]))); + } + UString focusedName; + const int focusedItem = _list.GetFocusedItem(); + if (focusedItem >= 0) + { + const int realIndex = GetRealItemIndex(focusedItem); + if (realIndex != kParentIndex) + focusedName = fs2us(Get_MainFileName_for_realIndex((unsigned)realIndex)); + } + const UString dirPathTemp = DirPrefix; + return Reload(dirPathTemp, selected, focusedName); +} + + +void CBrowseDialog2::OpenParentFolder() +{ +#if 1 // 0 : for debug + // we don't allow to go to parent of TempFolder. + // if (TempMode) + { + if (IsExactTempFolder(DirPrefix)) + return; + } +#endif + + UString parent, selected; + if (GetParentPath(DirPrefix, parent, selected)) + Reload(parent, selected); +} + + +void CBrowseDialog2::OnItemEnter() +{ + const bool alt = IsKeyDown(VK_MENU); + const bool ctrl = IsKeyDown(VK_CONTROL); + const bool shift = IsKeyDown(VK_SHIFT); + + const int index = _list.GetNextSelectedItem(-1); + if (index < 0) + return; + if (_list.GetNextSelectedItem(index) >= 0) + return; // more than one selected + const int realIndex = GetRealItemIndex(index); + if (realIndex == kParentIndex) + OpenParentFolder(); + else + { + const CBrowseItem &bi = _items[realIndex]; + const CFileInfo &file = _files[bi.MainFileIndex]; + if (alt) + { + Show_FileProps_Window(file); + return; + } + if (file.HasReparsePoint()) + { + // we don't want Link open operation, + // because user can think that it's usual folder/file (non-link). + ShowErrorMessage(*this, k_Message_Link_operation_was_Blocked); + return; + } + bool needExternal = true; + if (file.IsDir()) + { + if (!shift || alt || ctrl) // open folder in Explorer: + needExternal = false; + } + const UString fullPath = DirPrefix + fs2us(file.Name); + if (needExternal) + { + StartApplicationDontWait(DirPrefix, fullPath, (HWND)*this); + return; + } + UString s = fullPath; + s.Add_PathSepar(); + const HRESULT res = Reload(s, UString()); + if (res != S_OK) + MessageBox_HResError(*this, res, s); + // SetPathEditText(); + } +} + + +void MyBrowseForTempFolder(HWND owner) +{ + FString tempPathF; + if (!NFile::NDir::MyGetTempPath(tempPathF) || tempPathF.IsEmpty()) + { + MessageBox_LastError_path(owner, tempPathF); + return; + } + CBrowseDialog2 dialog; + + LangString_OnlyFromLangFile(IDM_TEMP_DIR, dialog.Title); + dialog.Title.Replace(L"...", L""); + if (dialog.Title.IsEmpty()) + dialog.Title = "Delete Temporary Files"; + + dialog.TempFolderPath = fs2us(tempPathF); + dialog.Create(owner); + // we can exit from dialog with 2 ways: + // IDCANCEL : Esc Key, or close icons + // IDCLOSE : with Close button +} diff --git a/CPP/7zip/UI/FileManager/BrowseDialog2.h b/CPP/7zip/UI/FileManager/BrowseDialog2.h new file mode 100644 index 0000000..6d5fb23 --- /dev/null +++ b/CPP/7zip/UI/FileManager/BrowseDialog2.h @@ -0,0 +1,10 @@ +// BrowseDialog2.h + +#ifndef ZIP7_INC_BROWSE_DIALOG2_H +#define ZIP7_INC_BROWSE_DIALOG2_H + +#include "../../../Windows/Window.h" + +void MyBrowseForTempFolder(HWND owner); + +#endif diff --git a/CPP/7zip/UI/FileManager/BrowseDialog2.rc b/CPP/7zip/UI/FileManager/BrowseDialog2.rc new file mode 100644 index 0000000..2ce2e45 --- /dev/null +++ b/CPP/7zip/UI/FileManager/BrowseDialog2.rc @@ -0,0 +1,32 @@ +#include "BrowseDialog2Res.h" +#include "../../GuiCommon.rc" +// #include "resource.h" + +#define xc 450 +#define yc 328 + +#define k_BROWSE2_y_CtrlSize 14 + +#define k_BROWSE2_y_List 40 + +IDD_BROWSE2 DIALOG 0, 0, xs, ys MY_MODAL_RESIZE_DIALOG_STYLE MY_FONT +CAPTION "7-Zip: Browse Temp Files" +{ +// EDITTEXT IDE_BROWSE_PATH, m, by - m - k_BROWSE_y_CtrlSize - k_BROWSE_y_CtrlSize - m, xc, k_BROWSE_y_CtrlSize, ES_AUTOHSCROLL + COMBOBOX IDC_BROWSE2_FILTER, m, by - m - k_BROWSE2_y_CtrlSize, xc, 30, MY_COMBO + +// DON'T USE DEFPUSHBUTTON here, because we use IDOK as default action for Enter key. + PUSHBUTTON "Close", IDCLOSE, bx2, by, bxs, bys, WS_GROUP + PUSHBUTTON "Help", IDHELP, bx1, by, bxs, bys + + PUSHBUTTON "Delete", IDS_BUTTON_DELETE, m , m, 64, bys + PUSHBUTTON "Refresh", IDM_VIEW_REFRESH, m + 64 + 8, m, 64, bys + + PUSHBUTTON "<--", IDB_BROWSE2_PARENT, m, m + 21, 24, bys +// LTEXT "", IDT_BROWSE_FOLDER, m + 30, m + 24, xc - 30, 8 + EDITTEXT IDT_BROWSE2_FOLDER, m + 28, m + 22, xc - 28, 14, ES_AUTOHSCROLL | ES_READONLY + + CONTROL "List1", IDL_BROWSE2, "SysListView32", + LVS_REPORT | LVS_SHOWSELALWAYS | LVS_SHAREIMAGELISTS | WS_BORDER | WS_TABSTOP, + m, m + k_BROWSE2_y_List, xc, yc - bys - m - k_BROWSE2_y_List - k_BROWSE2_y_CtrlSize - m +} diff --git a/CPP/7zip/UI/FileManager/BrowseDialog2Res.h b/CPP/7zip/UI/FileManager/BrowseDialog2Res.h new file mode 100644 index 0000000..add9158 --- /dev/null +++ b/CPP/7zip/UI/FileManager/BrowseDialog2Res.h @@ -0,0 +1,9 @@ +#define IDD_BROWSE2 93 + +#define IDL_BROWSE2 100 +#define IDT_BROWSE2_FOLDER 101 +// #define IDE_BROWSE2_PATH 102 +#define IDC_BROWSE2_FILTER 103 + +#define IDB_BROWSE2_PARENT 110 +// #define IDB_BROWSE2_CREATE_DIR 112 diff --git a/CPP/7zip/UI/FileManager/EditPage.rc b/CPP/7zip/UI/FileManager/EditPage.rc index 38f74ea..56c1ec0 100644 --- a/CPP/7zip/UI/FileManager/EditPage.rc +++ b/CPP/7zip/UI/FileManager/EditPage.rc @@ -1,8 +1,8 @@ #include "EditPageRes.h" #include "../../GuiCommon.rc" -#define xc 240 -#define yc 80 +#define xc OPTIONS_PAGE_XC_SIZE +#define yc 100 IDD_EDIT MY_PAGE #include "EditPage2.rc" diff --git a/CPP/7zip/UI/FileManager/ExtractCallback.cpp b/CPP/7zip/UI/FileManager/ExtractCallback.cpp index f674044..093534b 100644 --- a/CPP/7zip/UI/FileManager/ExtractCallback.cpp +++ b/CPP/7zip/UI/FileManager/ExtractCallback.cpp @@ -31,12 +31,15 @@ #ifndef Z7_NO_CRYPTO #include "PasswordDialog.h" #endif +#include "MemDialog.h" #include "PropertyName.h" using namespace NWindows; using namespace NFile; using namespace NFind; +extern bool g_DisableUserQuestions; + CExtractCallbackImp::~CExtractCallbackImp() {} void CExtractCallbackImp::Init() @@ -60,6 +63,13 @@ void CExtractCallbackImp::AddError_Message(LPCWSTR s) ProgressDialog->Sync.AddError_Message(s); } +void CExtractCallbackImp::AddError_Message_ShowArcPath(LPCWSTR s) +{ + Add_ArchiveName_Error(); + AddError_Message(s); +} + + #ifndef Z7_SFX Z7_COM7F_IMF(CExtractCallbackImp::SetNumFiles(UInt64 numFiles)) @@ -371,11 +381,11 @@ Z7_COM7F_IMF(CExtractCallbackImp::SetOperationResult(Int32 opRes, Int32 encrypte { UString s; SetExtractErrorMessage(opRes, encrypted, _currentFilePath, s); - Add_ArchiveName_Error(); - AddError_Message(s); + AddError_Message_ShowArcPath(s); } } + _currentFilePath.Empty(); #ifndef Z7_SFX if (_isFolder) NumFolders++; @@ -393,8 +403,7 @@ Z7_COM7F_IMF(CExtractCallbackImp::ReportExtractResult(Int32 opRes, Int32 encrypt { UString s; SetExtractErrorMessage(opRes, encrypted, name, s); - Add_ArchiveName_Error(); - AddError_Message(s); + AddError_Message_ShowArcPath(s); } return S_OK; } @@ -404,11 +413,12 @@ Z7_COM7F_IMF(CExtractCallbackImp::ReportExtractResult(Int32 opRes, Int32 encrypt HRESULT CExtractCallbackImp::BeforeOpen(const wchar_t *name, bool /* testMode */) { + _currentArchivePath = name; + _needWriteArchivePath = true; #ifndef Z7_SFX RINOK(ProgressDialog->Sync.CheckStop()) ProgressDialog->Sync.Set_TitleFileName(name); #endif - _currentArchivePath = name; return S_OK; } @@ -530,7 +540,7 @@ static UString GetBracedType(const wchar_t *type) { UString s ('['); s += type; - s += ']'; + s.Add_Char(']'); return s; } @@ -641,6 +651,10 @@ void CExtractCallbackImp::Add_ArchiveName_Error() HRESULT CExtractCallbackImp::ExtractResult(HRESULT result) { + #ifndef Z7_SFX + ProgressDialog->Sync.Set_FilePath(L""); + #endif + if (result == S_OK) return result; NumArchiveErrors++; @@ -834,11 +848,11 @@ Z7_COM7F_IMF(CExtractCallbackImp::GetStream7(const wchar_t *name, COM_TRY_BEGIN *outStream = NULL; _newVirtFileWasAdded = false; - _hashStreamWasUsed = false; + _hashStream_WasUsed = false; _needUpdateStat = false; if (_hashStream) - _hashStreamSpec->ReleaseStream(); + _hashStream->ReleaseStream(); GetItemBoolProp(getProp, kpidIsAltStream, _isAltStream); @@ -903,10 +917,10 @@ Z7_COM7F_IMF(CExtractCallbackImp::GetStream7(const wchar_t *name, if (_hashStream) { { - _hashStreamSpec->SetStream(outStreamLoc); + _hashStream->SetStream(outStreamLoc); outStreamLoc = _hashStream; - _hashStreamSpec->Init(true); - _hashStreamWasUsed = true; + _hashStream->Init(true); + _hashStream_WasUsed = true; } } @@ -951,13 +965,13 @@ Z7_COM7F_IMF(CExtractCallbackImp::SetOperationResult8(Int32 opRes, Int32 encrypt // _curSize_Defined = true; RINOK(VirtFileSystemSpec->CloseMemFile()) } - if (_hashStream && _hashStreamWasUsed) + if (_hashStream && _hashStream_WasUsed) { - _hashStreamSpec->_hash->Final(_isFolder, _isAltStream, _filePath); - _curSize = _hashStreamSpec->GetSize(); + _hashStream->_hash->Final(_isFolder, _isAltStream, _filePath); + _curSize = _hashStream->GetSize(); _curSize_Defined = true; - _hashStreamSpec->ReleaseStream(); - _hashStreamWasUsed = false; + _hashStream->ReleaseStream(); + _hashStream_WasUsed = false; } else if (_hashCalc && _needUpdateStat) { @@ -969,6 +983,114 @@ Z7_COM7F_IMF(CExtractCallbackImp::SetOperationResult8(Int32 opRes, Int32 encrypt } +Z7_COM7F_IMF(CExtractCallbackImp::RequestMemoryUse( + UInt32 flags, UInt32 indexType, UInt32 /* index */, const wchar_t *path, + UInt64 requiredSize, UInt64 *allowedSize, UInt32 *answerFlags)) +{ + UInt32 limit_GB = (UInt32)((*allowedSize + ((1u << 30) - 1)) >> 30); + + if ((flags & NRequestMemoryUseFlags::k_IsReport) == 0) + { + UInt64 limit_bytes = *allowedSize; + const UInt32 limit_GB_Registry = NExtract::Read_LimitGB(); + if (limit_GB_Registry != 0 && limit_GB_Registry != (UInt32)(Int32)-1) + { + const UInt64 limit_bytes_Registry = (UInt64)limit_GB_Registry << 30; + // registry_WasForced = true; + if ((flags & NRequestMemoryUseFlags::k_AllowedSize_WasForced) == 0 + || limit_bytes < limit_bytes_Registry) + { + limit_bytes = limit_bytes_Registry; + limit_GB = limit_GB_Registry; + } + } + *allowedSize = limit_bytes; + if (requiredSize <= limit_bytes) + { + *answerFlags = NRequestMemoryAnswerFlags::k_Allow; + return S_OK; + } + // default answer can be k_Allow, if limit was not forced, + // so we change answer to non-allowed here, + // because user has chance to change limit in GUI. + *answerFlags = NRequestMemoryAnswerFlags::k_Limit_Exceeded; + if (flags & NRequestMemoryUseFlags::k_SkipArc_IsExpected) + *answerFlags |= NRequestMemoryAnswerFlags::k_SkipArc; + } + + const UInt32 required_GB = (UInt32)((requiredSize + ((1u << 30) - 1)) >> 30); + + CMemDialog dialog; + dialog.Limit_GB = limit_GB; + dialog.Required_GB = required_GB; + dialog.TestMode = TestMode; + if (MultiArcMode) + dialog.ArcPath = _currentArchivePath; + if (path) + dialog.FilePath = path; + + if (!g_DisableUserQuestions + && (flags & NRequestMemoryUseFlags::k_IsReport) == 0) + { + if (_remember) + dialog.SkipArc = _skipArc; + else + { + dialog.ShowRemember = + (MultiArcMode + || indexType != NArchive::NEventIndexType::kNoIndex + || path); + ProgressDialog->WaitCreating(); + if (dialog.Create(*ProgressDialog) != IDCONTINUE) + { + *answerFlags = NRequestMemoryAnswerFlags::k_Stop; + return E_ABORT; + } + if (dialog.NeedSave) + NExtract::Save_LimitGB(dialog.Limit_GB); + if (dialog.Remember) + { + _remember = true; + _skipArc = dialog.SkipArc; + } + } + + *allowedSize = (UInt64)dialog.Limit_GB << 30; + if (!dialog.SkipArc) + { + *answerFlags = NRequestMemoryAnswerFlags::k_Allow; + return S_OK; + } + *answerFlags = + NRequestMemoryAnswerFlags::k_SkipArc + | NRequestMemoryAnswerFlags::k_Limit_Exceeded; + flags |= NRequestMemoryUseFlags::k_Report_SkipArc; + } + + if ((flags & NRequestMemoryUseFlags::k_NoErrorMessage) == 0) + { + UString s ("ERROR: "); + dialog.AddInfoMessage_To_String(s); + s.Add_LF(); + // if (indexType == NArchive::NEventIndexType::kNoIndex) + if ((flags & NRequestMemoryUseFlags::k_SkipArc_IsExpected) || + (flags & NRequestMemoryUseFlags::k_Report_SkipArc)) + s += LangString(IDS_MSG_ARC_UNPACKING_WAS_SKIPPED); +/* + else + s += LangString(IDS_MSG_ARC_FILES_UNPACKING_WAS_SKIPPED); +*/ + AddError_Message_ShowArcPath(s); + } + +/* + if ((flags & NRequestMemoryUseFlags::k_IsReport) == 0) + *answerFlags |= NRequestMemoryAnswerFlags::k_Limit_Exceeded; +*/ + return S_OK; +} + + // static const UInt32 kBlockSize = ((UInt32)1 << 31); @@ -1010,25 +1132,23 @@ Z7_COM7F_IMF(CVirtFileSystem::Write(const void *data, UInt32 size, UInt32 *proce _fileMode = true; } RINOK(FlushToDisk(false)) - return _outFileStream->Write(data, size, processedSize); + return _outFileStream.Interface()->Write(data, size, processedSize); } + HRESULT CVirtFileSystem::FlushToDisk(bool closeLast) { - if (!_outFileStream) - { - _outFileStreamSpec = new COutFileStream; - _outFileStream = _outFileStreamSpec; - } + _outFileStream.Create_if_Empty(); while (_numFlushed < Files.Size()) { const CVirtFile &file = Files[_numFlushed]; const FString path = DirPrefix + us2fs(Get_Correct_FsFile_Name(file.Name)); if (!_fileIsOpen) { - if (!_outFileStreamSpec->Create(path, false)) + if (!_outFileStream->Create_NEW(path)) { - _outFileStream.Release(); + // do we need to release stream here? + // _outFileStream.Release(); return E_FAIL; // MessageBoxMyError(UString("Can't create file ") + fs2us(tempFilePath)); } @@ -1040,13 +1160,17 @@ HRESULT CVirtFileSystem::FlushToDisk(bool closeLast) if (file.CTimeDefined || file.ATimeDefined || file.MTimeDefined) - _outFileStreamSpec->SetTime( + _outFileStream->SetTime( file.CTimeDefined ? &file.CTime : NULL, file.ATimeDefined ? &file.ATime : NULL, file.MTimeDefined ? &file.MTime : NULL); - _outFileStreamSpec->Close(); + _outFileStream->Close(); _numFlushed++; _fileIsOpen = false; + + if (ZoneBuf.Size() != 0) + WriteZoneFile_To_BaseFile(path, ZoneBuf); + if (file.AttribDefined) NDir::SetFileAttrib_PosixHighDetect(path, file.Attrib); } diff --git a/CPP/7zip/UI/FileManager/ExtractCallback.h b/CPP/7zip/UI/FileManager/ExtractCallback.h index c2aa470..daef5ec 100644 --- a/CPP/7zip/UI/FileManager/ExtractCallback.h +++ b/CPP/7zip/UI/FileManager/ExtractCallback.h @@ -103,13 +103,14 @@ Z7_CLASS_IMP_NOQIB_1( unsigned _numFlushed; bool _fileIsOpen; bool _fileMode; - COutFileStream *_outFileStreamSpec; - CMyComPtr _outFileStream; + CMyComPtr2 _outFileStream; public: CObjectVector Files; UInt64 MaxTotalAllocSize; FString DirPrefix; - + CByteBuffer ZoneBuf; + + CVirtFile &AddNewFile() { if (!Files.IsEmpty()) @@ -143,7 +144,9 @@ Z7_CLASS_IMP_NOQIB_1( size_t GetMemStreamWrittenSize() const { return _pos; } - CVirtFileSystem(): _outFileStreamSpec(NULL), MaxTotalAllocSize((UInt64)0 - 1) {} + CVirtFileSystem(): + MaxTotalAllocSize((UInt64)0 - 1) + {} void Init() { @@ -161,6 +164,8 @@ Z7_CLASS_IMP_NOQIB_1( #endif + + class CExtractCallbackImp Z7_final: public IFolderArchiveExtractCallback, /* IExtractCallbackUI: @@ -174,6 +179,7 @@ class CExtractCallbackImp Z7_final: public IFolderOperationsExtractCallback, public IFolderExtractToStreamCallback, public ICompressProgressInfo, + public IArchiveRequestMemoryUseCallback, #endif #ifndef Z7_NO_CRYPTO public ICryptoGetTextPassword, @@ -186,6 +192,7 @@ class CExtractCallbackImp Z7_final: Z7_COM_QI_ENTRY(IFolderOperationsExtractCallback) Z7_COM_QI_ENTRY(IFolderExtractToStreamCallback) Z7_COM_QI_ENTRY(ICompressProgressInfo) + Z7_COM_QI_ENTRY(IArchiveRequestMemoryUseCallback) #endif #ifndef Z7_NO_CRYPTO Z7_COM_QI_ENTRY(ICryptoGetTextPassword) @@ -202,65 +209,70 @@ class CExtractCallbackImp Z7_final: Z7_IFACE_COM7_IMP(IFolderOperationsExtractCallback) Z7_IFACE_COM7_IMP(IFolderExtractToStreamCallback) Z7_IFACE_COM7_IMP(ICompressProgressInfo) + Z7_IFACE_COM7_IMP(IArchiveRequestMemoryUseCallback) #endif #ifndef Z7_NO_CRYPTO Z7_IFACE_COM7_IMP(ICryptoGetTextPassword) #endif - - UString _currentArchivePath; bool _needWriteArchivePath; - bool _isFolder; - UString _currentFilePath; - UString _filePath; + bool _totalFilesDefined; + bool _totalBytesDefined; +public: + bool MultiArcMode; + bool ProcessAltStreams; + bool StreamMode; + bool ThereAreMessageErrors; +#ifndef Z7_NO_CRYPTO + bool PasswordIsDefined; + bool PasswordWasAsked; +#endif - #ifndef Z7_SFX +private: +#ifndef Z7_SFX bool _needUpdateStat; bool _newVirtFileWasAdded; bool _isAltStream; - bool _curSize_Defined; - UInt64 _curSize; // bool _extractMode; // bool _testMode; - bool _hashStreamWasUsed; - COutStreamWithHash *_hashStreamSpec; - CMyComPtr _hashStream; - IHashCalc *_hashCalc; // it's for stat in Test operation - #endif + bool _hashStream_WasUsed; + bool _curSize_Defined; + bool NeedAddFile; - HRESULT SetCurrentFilePath2(const wchar_t *filePath); - void AddError_Message(LPCWSTR message); - HRESULT MessageError(const char *message, const FString &path); - void Add_ArchiveName_Error(); + bool _remember; + bool _skipArc; +#endif + + UString _currentArchivePath; + UString _currentFilePath; + UString _filePath; + +#ifndef Z7_SFX + UInt64 _curSize; + CMyComPtr2 _hashStream; + IHashCalc *_hashCalc; // it's for stat in Test operation +#endif public: + CProgressDialog *ProgressDialog; - #ifndef Z7_SFX +#ifndef Z7_SFX CVirtFileSystem *VirtFileSystemSpec; CMyComPtr VirtFileSystem; - #endif - - bool ProcessAltStreams; - - bool StreamMode; - - CProgressDialog *ProgressDialog; - #ifndef Z7_SFX UInt64 NumFolders; UInt64 NumFiles; - bool NeedAddFile; - #endif +#endif + UInt32 NumArchiveErrors; - bool ThereAreMessageErrors; NExtract::NOverwriteMode::EEnum OverwriteMode; - #ifndef Z7_NO_CRYPTO - bool PasswordIsDefined; - bool PasswordWasAsked; - UString Password; - #endif + bool YesToAll; + bool TestMode; +#ifndef Z7_NO_CRYPTO + UString Password; +#endif UString _lang_Extracting; UString _lang_Testing; @@ -268,29 +280,35 @@ class CExtractCallbackImp Z7_final: UString _lang_Reading; UString _lang_Empty; - bool _totalFilesDefined; - bool _totalBytesDefined; - bool MultiArcMode; - CExtractCallbackImp(): - #ifndef Z7_SFX - _hashCalc(NULL), - #endif - ProcessAltStreams(true), - StreamMode(false), - OverwriteMode(NExtract::NOverwriteMode::kAsk), - #ifndef Z7_NO_CRYPTO - PasswordIsDefined(false), - PasswordWasAsked(false), - #endif - _totalFilesDefined(false), - _totalBytesDefined(false), - MultiArcMode(false) + _totalFilesDefined(false) + , _totalBytesDefined(false) + , MultiArcMode(false) + , ProcessAltStreams(true) + , StreamMode(false) +#ifndef Z7_NO_CRYPTO + , PasswordIsDefined(false) + , PasswordWasAsked(false) +#endif +#ifndef Z7_SFX + , _remember(false) + , _skipArc(false) + , _hashCalc(NULL) +#endif + , OverwriteMode(NExtract::NOverwriteMode::kAsk) + , YesToAll(false) + , TestMode(false) {} ~CExtractCallbackImp(); void Init(); + HRESULT SetCurrentFilePath2(const wchar_t *filePath); + void AddError_Message(LPCWSTR message); + void AddError_Message_ShowArcPath(LPCWSTR message); + HRESULT MessageError(const char *message, const FString &path); + void Add_ArchiveName_Error(); + #ifndef Z7_SFX void SetHashCalc(IHashCalc *hashCalc) { _hashCalc = hashCalc; } @@ -298,9 +316,8 @@ class CExtractCallbackImp Z7_final: { if (!hash) return; - _hashStreamSpec = new COutStreamWithHash; - _hashStream = _hashStreamSpec; - _hashStreamSpec->_hash = hash; + _hashStream.Create_if_Empty(); + _hashStream->_hash = hash; } #endif diff --git a/CPP/7zip/UI/FileManager/FM.cpp b/CPP/7zip/UI/FileManager/FM.cpp index 13189a7..fe4f2bd 100644 --- a/CPP/7zip/UI/FileManager/FM.cpp +++ b/CPP/7zip/UI/FileManager/FM.cpp @@ -47,6 +47,10 @@ using namespace NFind; // #define MAX_LOADSTRING 100 +extern +bool g_DisableUserQuestions; +bool g_DisableUserQuestions; + extern bool g_RAM_Size_Defined; bool g_RAM_Size_Defined; @@ -78,8 +82,9 @@ void FreeGlobalCodecs(); #ifndef UNDER_CE -extern -DWORD g_ComCtl32Version; +#ifdef Z7_USE_DYN_ComCtl32Version +Z7_DIAGNOSTIC_IGNORE_CAST_FUNCTION + DWORD g_ComCtl32Version; static DWORD GetDllVersion(LPCTSTR dllName) @@ -106,6 +111,7 @@ static DWORD GetDllVersion(LPCTSTR dllName) return dwVersion; } +#endif #endif bool g_IsSmallScreen = false; @@ -202,7 +208,36 @@ static const wchar_t * const kWindowClass = L"7-Zip::FM"; WS_MAXIMIZEBOX) #endif -// FUNCTION: InitInstance(HANDLE, int) + +/* +typedef HRESULT (WINAPI *Func_SetWindowTheme)( + HWND hwnd, + LPCWSTR pszSubAppName, + LPCWSTR pszSubIdList +); + +typedef BOOL (WINAPI *Func_AllowDarkModeForWindow)( + HWND a_HWND, BOOL a_Allow); + +enum PreferredAppMode +{ + Default, + AllowDark, + ForceDark, + ForceLight, + Max +}; +// ordinal 135, in 1903 +typedef BOOL (WINAPI *Func_SetPreferredAppMode)(PreferredAppMode appMode); + +typedef HRESULT (WINAPI *Func_DwmSetWindowAttribute)( + HWND hwnd, + DWORD dwAttribute, + LPCVOID pvAttribute, + DWORD cbAttribute +); +*/ + static BOOL InitInstance(int nCmdShow) { CWindow wnd; @@ -293,6 +328,69 @@ static BOOL InitInstance(int nCmdShow) x, y, xSize, ySize, NULL, NULL, g_hInstance, NULL)) return FALSE; + /* + // doesn't work + { + const HMODULE hmodule = LoadLibrary("UxTheme.dll"); + if (hmodule) + { + { + const + Func_AllowDarkModeForWindow f = Z7_GET_PROC_ADDRESS( + Func_AllowDarkModeForWindow, hmodule, + MAKEINTRESOURCEA(133)); + if (f) + { + BOOL res = f((HWND)wnd, TRUE); + res = res; + } + } + { + const + Func_SetPreferredAppMode f = Z7_GET_PROC_ADDRESS( + Func_SetPreferredAppMode, hmodule, + MAKEINTRESOURCEA(135)); + if (f) + { + f(ForceDark); + } + } + { + const + Func_SetWindowTheme f = Z7_GET_PROC_ADDRESS( + Func_SetWindowTheme, hmodule, + "SetWindowTheme"); + if (f) + { + // HRESULT hres = f((HWND)wnd, L"DarkMode_Explorer", NULL); + HRESULT hres = f((HWND)wnd, L"Explorer", NULL); + hres = hres; + } + } + FreeLibrary(hmodule); + } + } + { + const HMODULE hmodule = LoadLibrary("Dwmapi.dll"); + if (hmodule) + { + const + Func_DwmSetWindowAttribute f = Z7_GET_PROC_ADDRESS( + Func_DwmSetWindowAttribute, hmodule, + "DwmSetWindowAttribute"); + if (f) + { + #ifndef Z7_WIN_DWMWA_USE_IMMERSIVE_DARK_MODE + #define Z7_WIN_DWMWA_USE_IMMERSIVE_DARK_MODE 20 + #endif + BOOL value = TRUE; + f((HWND)wnd, Z7_WIN_DWMWA_USE_IMMERSIVE_DARK_MODE, &value, sizeof(value)); + } + FreeLibrary(hmodule); + } + } + */ + if (nCmdShow == SW_SHOWNORMAL || nCmdShow == SW_SHOW #ifndef UNDER_CE @@ -519,10 +617,10 @@ static int WINAPI WinMain2(int nCmdShow) InitCommonControls(); - #ifndef UNDER_CE +#ifdef Z7_USE_DYN_ComCtl32Version g_ComCtl32Version = ::GetDllVersion(TEXT("comctl32.dll")); g_LVN_ITEMACTIVATE_Support = (g_ComCtl32Version >= MAKELONG(71, 4)); - #endif +#endif #if defined(_WIN32) && !defined(_WIN64) && !defined(UNDER_CE) Set_Wow64(); diff --git a/CPP/7zip/UI/FileManager/FM.dsp b/CPP/7zip/UI/FileManager/FM.dsp index 1ae054d..4c549d6 100644 --- a/CPP/7zip/UI/FileManager/FM.dsp +++ b/CPP/7zip/UI/FileManager/FM.dsp @@ -475,6 +475,14 @@ SOURCE=.\BrowseDialog.h # End Source File # Begin Source File +SOURCE=.\BrowseDialog2.cpp +# End Source File +# Begin Source File + +SOURCE=.\BrowseDialog2.h +# End Source File +# Begin Source File + SOURCE=.\ComboDialog.cpp # End Source File # Begin Source File @@ -519,6 +527,14 @@ SOURCE=.\ListViewDialog.h # End Source File # Begin Source File +SOURCE=.\MemDialog.cpp +# End Source File +# Begin Source File + +SOURCE=.\MemDialog.h +# End Source File +# Begin Source File + SOURCE=MessagesDialog.cpp # End Source File # Begin Source File @@ -707,6 +723,10 @@ SOURCE=..\..\..\..\C\7zTypes.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\7zVersion.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\7zWindows.h # End Source File # Begin Source File @@ -742,6 +762,10 @@ SOURCE=..\..\..\..\C\DllSecur.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\Precomp.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Sort.c # SUBTRACT CPP /YX /Yc /Yu # End Source File @@ -1068,10 +1092,18 @@ SOURCE=..\..\..\Windows\Window.h # PROP Default_Filter "" # Begin Source File +SOURCE=..\..\..\Common\AutoPtr.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Common\Common.h # End Source File # Begin Source File +SOURCE=..\..\..\Common\Common0.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Common\ComTry.h # End Source File # Begin Source File diff --git a/CPP/7zip/UI/FileManager/FM.mak b/CPP/7zip/UI/FileManager/FM.mak index 8331285..d8e2469 100644 --- a/CPP/7zip/UI/FileManager/FM.mak +++ b/CPP/7zip/UI/FileManager/FM.mak @@ -5,7 +5,8 @@ CFLAGS = $(CFLAGS) \ LIBS = $(LIBS) ceshell.lib Commctrl.lib !ELSE LIBS = $(LIBS) comctl32.lib htmlhelp.lib comdlg32.lib Mpr.lib Gdi32.lib -CFLAGS = $(CFLAGS) -DZ7_LONG_PATH -DZ7_DEVICE_FILE +CFLAGS = $(CFLAGS) -DZ7_DEVICE_FILE +# -DZ7_LONG_PATH LFLAGS = $(LFLAGS) /DELAYLOAD:mpr.dll LIBS = $(LIBS) delayimp.lib !ENDIF @@ -13,6 +14,7 @@ LIBS = $(LIBS) delayimp.lib FM_OBJS = \ $O\App.obj \ $O\BrowseDialog.obj \ + $O\BrowseDialog2.obj \ $O\ClassDefs.obj \ $O\EnumFormatEtc.obj \ $O\ExtractCallback.obj \ @@ -25,6 +27,7 @@ FM_OBJS = \ $O\FSFolderCopy.obj \ $O\HelpUtils.obj \ $O\LangUtils.obj \ + $O\MemDialog.obj \ $O\MenuPage.obj \ $O\MyLoadMenu.obj \ $O\OpenCallback.obj \ diff --git a/CPP/7zip/UI/FileManager/FSDrives.cpp b/CPP/7zip/UI/FileManager/FSDrives.cpp index 985d7c4..19d0814 100644 --- a/CPP/7zip/UI/FileManager/FSDrives.cpp +++ b/CPP/7zip/UI/FileManager/FSDrives.cpp @@ -64,7 +64,7 @@ static HRESULT CopyFileSpec(CFSTR fromPath, CFSTR toPath, bool writeToDisk, UInt return GetLastError_noZero_HRESULT(); } else - if (!outFile.Create(toPath, true)) + if (!outFile.Create_ALWAYS(toPath)) return GetLastError_noZero_HRESULT(); CPhysTempBuffer tempBuffer; diff --git a/CPP/7zip/UI/FileManager/FSFolder.cpp b/CPP/7zip/UI/FileManager/FSFolder.cpp index 06c572d..26a2ccf 100644 --- a/CPP/7zip/UI/FileManager/FSFolder.cpp +++ b/CPP/7zip/UI/FileManager/FSFolder.cpp @@ -45,7 +45,7 @@ typedef struct _IO_STATUS_BLOCK { #include "SysIconUtils.h" -#if _WIN32_WINNT < 0x0501 +#if !defined(_WIN32_WINNT) || _WIN32_WINNT < 0x0501 #ifdef _APISETFILE_ // Windows SDK 8.1 defines in fileapi.h the function GetCompressedFileSizeW only if _WIN32_WINNT >= 0x0501 // But real support version for that function is NT 3.1 (probably) @@ -63,6 +63,7 @@ using namespace NDir; using namespace NName; #ifndef USE_UNICODE_FSTRING +int CompareFileNames_ForFolderList(const FChar *s1, const FChar *s2); int CompareFileNames_ForFolderList(const FChar *s1, const FChar *s2) { return CompareFileNames_ForFolderList(fs2us(s1), fs2us(s2)); @@ -119,7 +120,7 @@ HRESULT CFSFolder::Init(const FString &path /* , IFolderFolder *parentFolder */) CFindFile findFile; CFileInfo fi; FString path2 = _path; - path2 += '*'; // CHAR_ANY_MASK; + path2.Add_Char('*'); // CHAR_ANY_MASK; if (!findFile.FindFirst(path2, fi)) return lastError; } @@ -357,7 +358,7 @@ bool CFSFolder::SaveComments() attrib = fi.Attrib; } NIO::COutFile file; - if (!file.CreateAlways(path, attrib)) + if (!file.Create_ALWAYS_with_Attribs(path, attrib)) return false; UInt32 processed; file.Write(utf, utf.Len(), processed); @@ -467,7 +468,7 @@ typedef enum Z7_WIN_FILE_INFORMATION_CLASS; -#if (_WIN32_WINNT >= 0x0500) && !defined(_M_IA64) +#if defined(_WIN32_WINNT) && (_WIN32_WINNT >= 0x0500) && !defined(_M_IA64) #define Z7_WIN_NTSTATUS NTSTATUS #define Z7_WIN_IO_STATUS_BLOCK IO_STATUS_BLOCK #else @@ -495,6 +496,7 @@ EXTERN_C_END static Func_NtQueryInformationFile f_NtQueryInformationFile; static bool g_NtQueryInformationFile_WasRequested = false; +Z7_DIAGNOSTIC_IGNORE_CAST_FUNCTION void CFSFolder::ReadChangeTime(CDirItem &di) { @@ -907,7 +909,7 @@ Z7_COM7F_IMF(CFSFolder::BindToParentFolder(IFolderFolder **resultFolder)) int pos = _path.ReverseFind_PathSepar(); if (pos < 0 || pos != (int)_path.Len() - 1) return E_FAIL; - FString parentPath = _path.Left(pos); + FString parentPath = _path.Left((unsigned)pos); pos = parentPath.ReverseFind_PathSepar(); parentPath.DeleteFrom((unsigned)(pos + 1)); @@ -1071,7 +1073,7 @@ Z7_COM7F_IMF(CFSFolder::CreateFile(const wchar_t *name, IProgress * /* progress FString absPath; GetAbsPath(name, absPath); NIO::COutFile outFile; - if (!outFile.Create(absPath, false)) + if (!outFile.Create_NEW(absPath)) return GetLastError_noZero_HRESULT(); return S_OK; } diff --git a/CPP/7zip/UI/FileManager/FSFolderCopy.cpp b/CPP/7zip/UI/FileManager/FSFolderCopy.cpp index db4ae0a..67499fc 100644 --- a/CPP/7zip/UI/FileManager/FSFolderCopy.cpp +++ b/CPP/7zip/UI/FileManager/FSFolderCopy.cpp @@ -51,7 +51,7 @@ HRESULT CCopyStateIO::MyCopyFile(CFSTR inPath, CFSTR outPath, DWORD attrib) return S_OK; } - if (!outFile.Create(outPath, true)) + if (!outFile.Create_ALWAYS(outPath)) { ErrorFileIndex = 1; return S_OK; @@ -184,6 +184,12 @@ static DWORD CALLBACK CopyProgressRoutine( return (pi.ProgressResult == S_OK ? PROGRESS_CONTINUE : PROGRESS_CANCEL); } +#if !defined(Z7_WIN32_WINNT_MIN) || Z7_WIN32_WINNT_MIN < 0x0500 // win2000 +#define Z7_USE_DYN_MoveFileWithProgressW +#endif + +#ifdef Z7_USE_DYN_MoveFileWithProgressW +// nt4 typedef BOOL (WINAPI * Func_CopyFileExA)( IN LPCSTR lpExistingFileName, IN LPCSTR lpNewFileName, @@ -193,6 +199,7 @@ typedef BOOL (WINAPI * Func_CopyFileExA)( IN DWORD dwCopyFlags ); +// nt4 typedef BOOL (WINAPI * Func_CopyFileExW)( IN LPCWSTR lpExistingFileName, IN LPCWSTR lpNewFileName, @@ -202,6 +209,7 @@ typedef BOOL (WINAPI * Func_CopyFileExW)( IN DWORD dwCopyFlags ); +// win2000 typedef BOOL (WINAPI * Func_MoveFileWithProgressW)( IN LPCWSTR lpExistingFileName, IN LPCWSTR lpNewFileName, @@ -209,6 +217,7 @@ typedef BOOL (WINAPI * Func_MoveFileWithProgressW)( IN LPVOID lpData OPTIONAL, IN DWORD dwFlags ); +#endif struct CCopyState { @@ -218,6 +227,8 @@ struct CCopyState bool UseReadWriteMode; bool IsAltStreamsDest; +#ifdef Z7_USE_DYN_MoveFileWithProgressW +private: Func_CopyFileExW my_CopyFileExW; #ifndef UNDER_CE Func_MoveFileWithProgressW my_MoveFileWithProgressW; @@ -225,8 +236,10 @@ struct CCopyState #ifndef _UNICODE Func_CopyFileExA my_CopyFileExA; #endif +public: + CCopyState(); +#endif - void Prepare(); bool CopyFile_NT(const wchar_t *oldFile, const wchar_t *newFile); bool CopyFile_Sys(CFSTR oldFile, CFSTR newFile); bool MoveFile_Sys(CFSTR oldFile, CFSTR newFile); @@ -241,7 +254,9 @@ HRESULT CCopyState::CallProgress() return ProgressInfo.Progress->SetCompleted(&ProgressInfo.StartPos); } -void CCopyState::Prepare() +#ifdef Z7_USE_DYN_MoveFileWithProgressW + +CCopyState::CCopyState() { my_CopyFileExW = NULL; #ifndef UNDER_CE @@ -276,6 +291,8 @@ void CCopyState::Prepare() } } +#endif + /* WinXP-64: CopyFileW(fromFile, toFile:altStream) OK - there are NO alt streams in fromFile @@ -285,10 +302,20 @@ void CCopyState::Prepare() bool CCopyState::CopyFile_NT(const wchar_t *oldFile, const wchar_t *newFile) { BOOL cancelFlag = FALSE; +#ifdef Z7_USE_DYN_MoveFileWithProgressW if (my_CopyFileExW) - return BOOLToBool(my_CopyFileExW(oldFile, newFile, CopyProgressRoutine, +#endif + return BOOLToBool( +#ifdef Z7_USE_DYN_MoveFileWithProgressW + my_CopyFileExW +#else + CopyFileExW +#endif + (oldFile, newFile, CopyProgressRoutine, &ProgressInfo, &cancelFlag, COPY_FILE_FAIL_IF_EXISTS)); +#ifdef Z7_USE_DYN_MoveFileWithProgressW return BOOLToBool(::CopyFileW(oldFile, newFile, TRUE)); +#endif } bool CCopyState::CopyFile_Sys(CFSTR oldFile, CFSTR newFile) @@ -296,10 +323,18 @@ bool CCopyState::CopyFile_Sys(CFSTR oldFile, CFSTR newFile) #ifndef _UNICODE if (!g_IsNT) { +#ifdef Z7_USE_DYN_MoveFileWithProgressW if (my_CopyFileExA) +#endif { BOOL cancelFlag = FALSE; - if (my_CopyFileExA(fs2fas(oldFile), fs2fas(newFile), + if ( +#ifdef Z7_USE_DYN_MoveFileWithProgressW + my_CopyFileExA +#else + CopyFileExA +#endif + (fs2fas(oldFile), fs2fas(newFile), CopyProgressRoutine, &ProgressInfo, &cancelFlag, COPY_FILE_FAIL_IF_EXISTS)) return true; if (::GetLastError() != ERROR_CALL_NOT_IMPLEMENTED) @@ -336,11 +371,19 @@ bool CCopyState::MoveFile_Sys(CFSTR oldFile, CFSTR newFile) #ifndef UNDER_CE // if (IsItWindows2000orHigher()) // { +#ifdef Z7_USE_DYN_MoveFileWithProgressW if (my_MoveFileWithProgressW) +#endif { IF_USE_MAIN_PATH_2(oldFile, newFile) { - if (my_MoveFileWithProgressW(fs2us(oldFile), fs2us(newFile), CopyProgressRoutine, + if ( +#ifdef Z7_USE_DYN_MoveFileWithProgressW + my_MoveFileWithProgressW +#else + MoveFileWithProgressW +#endif + (fs2us(oldFile), fs2us(newFile), CopyProgressRoutine, &ProgressInfo, MOVEFILE_COPY_ALLOWED)) return true; } @@ -352,7 +395,13 @@ bool CCopyState::MoveFile_Sys(CFSTR oldFile, CFSTR newFile) UString superPathOld, superPathNew; if (!GetSuperPaths(oldFile, newFile, superPathOld, superPathNew, USE_MAIN_PATH_2)) return false; - if (my_MoveFileWithProgressW(superPathOld, superPathNew, CopyProgressRoutine, + if ( +#ifdef Z7_USE_DYN_MoveFileWithProgressW + my_MoveFileWithProgressW +#else + MoveFileWithProgressW +#endif + (superPathOld, superPathNew, CopyProgressRoutine, &ProgressInfo, MOVEFILE_COPY_ALLOWED)) return true; } @@ -645,7 +694,6 @@ Z7_COM7F_IMF(CFSFolder::CopyTo(Int32 moveMode, const UInt32 *indices, UInt32 num if there are alt streams in fromFile. So we don't use CopyFileW() for alt Streams. */ state.UseReadWriteMode = isAltDest; - state.Prepare(); for (i = 0; i < numItems; i++) { @@ -746,7 +794,6 @@ HRESULT CopyFileSystemItems( if there are alt streams in fromFile. So we don't use CopyFileW() for alt Streams. */ state.UseReadWriteMode = isAltDest; - state.Prepare(); FOR_VECTOR (i, itemsPaths) { diff --git a/CPP/7zip/UI/FileManager/FileFolderPluginOpen.cpp b/CPP/7zip/UI/FileManager/FileFolderPluginOpen.cpp index e4e9997..1c33464 100644 --- a/CPP/7zip/UI/FileManager/FileFolderPluginOpen.cpp +++ b/CPP/7zip/UI/FileManager/FileFolderPluginOpen.cpp @@ -166,7 +166,7 @@ static UString GetBracedType(const wchar_t *type) { UString s ('['); s += type; - s += ']'; + s.Add_Char(']'); return s; } diff --git a/CPP/7zip/UI/FileManager/FoldersPage.rc b/CPP/7zip/UI/FileManager/FoldersPage.rc index cb345ea..3317a08 100644 --- a/CPP/7zip/UI/FileManager/FoldersPage.rc +++ b/CPP/7zip/UI/FileManager/FoldersPage.rc @@ -1,7 +1,7 @@ #include "FoldersPageRes.h" #include "../../GuiCommon.rc" -#define xc 240 +#define xc OPTIONS_PAGE_XC_SIZE #define yc 100 IDD_FOLDERS MY_PAGE diff --git a/CPP/7zip/UI/FileManager/FoldersPage2.rc b/CPP/7zip/UI/FileManager/FoldersPage2.rc index 9b9276e..70f832e 100644 --- a/CPP/7zip/UI/FileManager/FoldersPage2.rc +++ b/CPP/7zip/UI/FileManager/FoldersPage2.rc @@ -1,16 +1,11 @@ CAPTION "Folders" BEGIN // GROUPBOX "&Working folder", IDT_FOLDERS_WORKING_FOLDER, m, m, xc, 98 - LTEXT "&Working folder", IDT_FOLDERS_WORKING_FOLDER, m, m, xc, 8 - CONTROL "&System temp folder", IDR_FOLDERS_WORK_SYSTEM, "Button", BS_AUTORADIOBUTTON | WS_GROUP, - m, 20, xc, 10 - CONTROL "&Current", IDR_FOLDERS_WORK_CURRENT, "Button", BS_AUTORADIOBUTTON, - m, 34, xc, 10 - CONTROL "Specified:", IDR_FOLDERS_WORK_SPECIFIED, "Button", BS_AUTORADIOBUTTON, - m, 48, xc, 10 + MY_CONTROL_AUTORADIOBUTTON_GROUP ( "&System temp folder", IDR_FOLDERS_WORK_SYSTEM, m, 20, xc) + MY_CONTROL_AUTORADIOBUTTON ( "&Current", IDR_FOLDERS_WORK_CURRENT, m, 34, xc) + MY_CONTROL_AUTORADIOBUTTON ( "Specified:", IDR_FOLDERS_WORK_SPECIFIED, m, 48, xc) EDITTEXT IDE_FOLDERS_WORK_PATH, m + m, 62, xc - m - m - bxsDots, 14, ES_AUTOHSCROLL PUSHBUTTON "...", IDB_FOLDERS_WORK_PATH, xs - m - bxsDots, 61, bxsDots, bys - CONTROL "Use for removable drives only", IDX_FOLDERS_WORK_FOR_REMOVABLE, MY_CHECKBOX, - m, 86, xc, 10 + MY_CONTROL_CHECKBOX ( "Use for removable drives only", IDX_FOLDERS_WORK_FOR_REMOVABLE, m, 86, xc) END diff --git a/CPP/7zip/UI/FileManager/LangPage.cpp b/CPP/7zip/UI/FileManager/LangPage.cpp index ec1dd2e..3aeaf13 100644 --- a/CPP/7zip/UI/FileManager/LangPage.cpp +++ b/CPP/7zip/UI/FileManager/LangPage.cpp @@ -16,7 +16,7 @@ using namespace NWindows; -static const unsigned k_NumLangLines_EN = 429; +static const unsigned k_NumLangLines_EN = 443; #ifdef Z7_LANG static const UInt32 kLangIDs[] = @@ -50,7 +50,7 @@ static void NativeLangString(UString &dest, const wchar_t *s) { dest += " ("; dest += s; - dest += ')'; + dest.Add_Char(')'); } bool LangOpen(CLang &lang, CFSTR fileName); diff --git a/CPP/7zip/UI/FileManager/LangPage.rc b/CPP/7zip/UI/FileManager/LangPage.rc index 506f102..60d730f 100644 --- a/CPP/7zip/UI/FileManager/LangPage.rc +++ b/CPP/7zip/UI/FileManager/LangPage.rc @@ -1,12 +1,12 @@ #include "LangPageRes.h" #include "../../GuiCommon.rc" -#define xc 240 -#define yc 252 +#define xc OPTIONS_PAGE_XC_SIZE +#define yc OPTIONS_PAGE_YC_SIZE #define y 32 -IDD_LANG DIALOG 0, 0, xs, ys MY_PAGE_STYLE MY_FONT +IDD_LANG DIALOG MY_PAGE_POSTFIX CAPTION "Language" { LTEXT "Language:", IDT_LANG_LANG, m, m, xc, 8 diff --git a/CPP/7zip/UI/FileManager/LangUtils.cpp b/CPP/7zip/UI/FileManager/LangUtils.cpp index 4c9d16f..8fcb507 100644 --- a/CPP/7zip/UI/FileManager/LangUtils.cpp +++ b/CPP/7zip/UI/FileManager/LangUtils.cpp @@ -56,6 +56,10 @@ void LangSetDlgItemText(HWND dialog, UInt32 controlID, UInt32 langID) } } +#ifndef IDCONTINUE +#define IDCONTINUE 11 +#endif + static const CIDLangPair kLangPairs[] = { { IDOK, 401 }, @@ -63,7 +67,8 @@ static const CIDLangPair kLangPairs[] = { IDYES, 406 }, { IDNO, 407 }, { IDCLOSE, 408 }, - { IDHELP, 409 } + { IDHELP, 409 }, + { IDCONTINUE, 411 } }; @@ -99,7 +104,7 @@ void LangSetDlgItems_Colon(HWND dialog, const UInt32 *ids, unsigned numItems) { CWindow window(GetDlgItem(dialog, (int)id)); UString s2 = s; - s2 += ':'; + s2.Add_Colon(); window.SetText(s2); } } @@ -163,7 +168,7 @@ void LangString_OnlyFromLangFile(UInt32 langID, UString &dest) static const char * const kLangs = "ar.bg.ca.zh.-tw.-cn.cs.da.de.el.en.es.fi.fr.he.hu.is." - "it.ja.ko.nl.no.=nb.=nn.pl.pt.-br.rm.ro.ru.sr.=hr.-spl.-spc.sk.sq.sv.th.tr." + "it.ja.ko.nl.no.=nb.=nn.pl.pt.-br.rm.ro.ru.sr.=hr.-spl.-spc.=hr.=bs.sk.sq.sv.th.tr." "ur.id.uk.be.sl.et.lv.lt.tg.fa.vi.hy.az.eu.hsb.mk." "st.ts.tn.ve.xh.zu.af.ka.fo.hi.mt.se.ga.yi.ms.kk." "ky.sw.tk.uz.-latn.-cyrl.tt.bn.pa.-in.gu.or.ta.te.kn.ml.as.mr.sa." @@ -198,7 +203,7 @@ static void FindShortNames(UInt32 primeLang, AStringVector &names) p++; } while (p != p2) - s += (char)(Byte)*p++; + s.Add_Char((char)(Byte)*p++); names.Add(s); } p = p2 + 1; @@ -238,10 +243,14 @@ void Lang_GetShortNames_for_DefaultLang(AStringVector &names, unsigned &subLang) { names.Clear(); subLang = 0; - const LANGID sysLang = GetSystemDefaultLangID(); // "Language for non-Unicode programs" in XP64 - const LANGID userLang = GetUserDefaultLangID(); // "Standards and formats" language in XP64 + // Region / Administative / Language for non-Unicode programs: + const LANGID sysLang = GetSystemDefaultLangID(); + + // Region / Formats / Format: + const LANGID userLang = GetUserDefaultLangID(); - if (sysLang != userLang) + if (PRIMARYLANGID(sysLang) != + PRIMARYLANGID(userLang)) return; const LANGID langID = userLang; diff --git a/CPP/7zip/UI/FileManager/MemDialog.cpp b/CPP/7zip/UI/FileManager/MemDialog.cpp new file mode 100644 index 0000000..1ba717e --- /dev/null +++ b/CPP/7zip/UI/FileManager/MemDialog.cpp @@ -0,0 +1,218 @@ +// MemDialog.cpp + +#include "StdAfx.h" + +#include + +#include "MemDialog.h" + +#include "../../../Common/StringToInt.h" +#include "../../../Windows/System.h" +#include "../../../Windows/ErrorMsg.h" + +#include "../Explorer/MyMessages.h" +#include "../GUI/ExtractRes.h" + +#include "resourceGui.h" + +#ifdef Z7_LANG +#include "LangUtils.h" +#endif + +#ifdef Z7_LANG +static const UInt32 kLangIDs[] = +{ + IDX_MEM_SAVE_LIMIT, + IDX_MEM_REMEMBER, + IDG_MEM_ACTION, + IDR_MEM_ACTION_ALLOW, + IDR_MEM_ACTION_SKIP_ARC + // , IDR_MEM_SKIP_FILE +}; +#endif + +static const unsigned k_Action_Buttons[] = +{ + IDR_MEM_ACTION_ALLOW, + IDR_MEM_ACTION_SKIP_ARC + // , IDR_MEM_SKIP_FILE +}; + + +void CMemDialog::EnableSpin(bool enable) +{ + EnableItem(IDC_MEM_SPIN, enable); + EnableItem(IDE_MEM_SPIN_EDIT, enable); +} + + +static void AddSize_GB(UString &s, UInt32 size_GB, UInt32 id) +{ + s.Add_LF(); + s += " "; + s.Add_UInt32(size_GB); + s += " GB : "; + AddLangString(s, id); +} + +void CMemDialog::AddInfoMessage_To_String(UString &s, UInt64 *ramSize_GB) +{ + AddLangString(s, IDS_MEM_REQUIRES_BIG_MEM); + AddSize_GB(s, Required_GB, IDS_MEM_REQUIRED_MEM_SIZE); + AddSize_GB(s, Limit_GB, IDS_MEM_CURRENT_MEM_LIMIT); + if (ramSize_GB) + AddSize_GB(s, (UInt32)*ramSize_GB, IDS_MEM_RAM_SIZE); + if (!FilePath.IsEmpty()) + { + s.Add_LF(); + s += "File: "; + s += FilePath; + } +} + +/* +int CMemDialog::AddAction(UINT id) +{ + const int index = (int)m_Action.AddString(LangString(id)); + m_Action.SetItemData(index, (LPARAM)id); + return index; +} +*/ + +bool CMemDialog::OnInit() +{ + #ifdef Z7_LANG + LangSetWindowText(*this, IDD_MEM); + LangSetDlgItems(*this, kLangIDs, Z7_ARRAY_SIZE(kLangIDs)); + #endif + + // m_Action.Attach(GetItem(IDC_MEM_ACTION)); + + UInt64 ramSize = (UInt64)sizeof(size_t) << 29; + const bool ramSize_defined = NWindows::NSystem::GetRamSize(ramSize); + // ramSize *= 10; // for debug + + UInt64 ramSize_GB = (ramSize + (1u << 29)) >> 30; + if (ramSize_GB == 0) + ramSize_GB = 1; + + const bool is_Allowed = (!ramSize_defined || ramSize > ((UInt64)Required_GB << 30)); + { + UString s; + if (!is_Allowed) + { + AddLangString(s, IDS_MEM_ERROR); + s.Add_LF(); + } + AddInfoMessage_To_String(s, is_Allowed ? NULL : &ramSize_GB); + if (!ArcPath.IsEmpty()) + // for (int i = 0; i < 10; i++) + { + s.Add_LF(); + AddLangString(s, TestMode ? + IDS_PROGRESS_TESTING : + IDS_PROGRESS_EXTRACTING); + s += ": "; + s += ArcPath; + } + SetItemText(IDT_MEM_MESSAGE, s); + + s = "GB"; + if (ramSize_defined) + { + s += " / "; + s.Add_UInt64(ramSize_GB); + s += " GB (RAM)"; + } + SetItemText(IDT_MEM_GB, s); + } + const UINT valMin = 1; + UINT valMax = 64; // 64GB for RAR7 + if (ramSize_defined /* && ramSize_GB > valMax */) + { + const UINT k_max_val = 1u << 14; + if (ramSize_GB >= k_max_val) + valMax = k_max_val; + else if (ramSize_GB > 1) + valMax = (UINT)ramSize_GB - 1; + else + valMax = 1; + } + + SendItemMessage(IDC_MEM_SPIN, UDM_SETRANGE, 0, MAKELPARAM(valMax, valMin)); // Sets the controls direction + // UDM_SETPOS doesn't set value larger than max value (valMax) of range: + SendItemMessage(IDC_MEM_SPIN, UDM_SETPOS, 0, Required_GB); + { + UString s; + s.Add_UInt32(Required_GB); + SetItemText(IDE_MEM_SPIN_EDIT, s); + } + + EnableSpin(false); + + /* + AddAction(IDB_ALLOW_OPERATION); + m_Action.SetCurSel(0); + AddAction(IDB_MEM_SKIP_ARC); + AddAction(IDB_MEM_SKIP_FILE); + */ + + const UINT buttonId = is_Allowed ? + IDR_MEM_ACTION_ALLOW : + IDR_MEM_ACTION_SKIP_ARC; + + CheckRadioButton( + k_Action_Buttons[0], + k_Action_Buttons[Z7_ARRAY_SIZE(k_Action_Buttons) - 1], + buttonId); + /* + if (!ShowSkipFile) + HideItem(IDR_MEM_SKIP_FILE); + */ + if (!ShowRemember) + HideItem(IDX_MEM_REMEMBER); + return CModalDialog::OnInit(); +} + + +bool CMemDialog::OnButtonClicked(unsigned buttonID, HWND buttonHWND) +{ + if (buttonID == IDX_MEM_SAVE_LIMIT) + { + EnableSpin(IsButtonCheckedBool(IDX_MEM_SAVE_LIMIT)); + return true; + } + return CDialog::OnButtonClicked(buttonID, buttonHWND); +} + + +void CMemDialog::OnContinue() +{ + Remember = IsButtonCheckedBool(IDX_MEM_REMEMBER); + NeedSave = IsButtonCheckedBool(IDX_MEM_SAVE_LIMIT); + SkipArc = IsButtonCheckedBool(IDR_MEM_ACTION_SKIP_ARC); + if (NeedSave) + { +#if 0 + // UDM_GETPOS doesn't support value outside of range that was set: + LRESULT lresult = SendItemMessage(IDC_MEM_SPIN, UDM_GETPOS, 0, 0); + const UInt32 val = LOWORD(lresult); + if (HIWORD(lresult) != 0) // the value outside of allowed range +#else + UString s; + GetItemText(IDE_MEM_SPIN_EDIT, s); + const wchar_t *end; + const UInt32 val = ConvertStringToUInt32(s.Ptr(), &end); + if (s.IsEmpty() || *end != 0 || val > (1u << 30)) +#endif + { + ShowErrorMessage(*this, + NWindows::NError::MyFormatMessage(E_INVALIDARG) + // L"Incorrect value" + ); + return; + } + Limit_GB = val; + } + CModalDialog::OnContinue(); +} diff --git a/CPP/7zip/UI/FileManager/MemDialog.h b/CPP/7zip/UI/FileManager/MemDialog.h new file mode 100644 index 0000000..79de658 --- /dev/null +++ b/CPP/7zip/UI/FileManager/MemDialog.h @@ -0,0 +1,48 @@ +// MemDialog.h + +#ifndef ZIP7_INC_MEM_DIALOG_H +#define ZIP7_INC_MEM_DIALOG_H + +#include "../../../Windows/Control/Dialog.h" +// #include "../../../Windows/Control/ComboBox.h" + +#include "MemDialogRes.h" + +class CMemDialog: public NWindows::NControl::CModalDialog +{ + // NWindows::NControl::CComboBox m_Action; + // we can disable default OnOK() when we press Enter + // virtual void OnOK() Z7_override { } + virtual void OnContinue() Z7_override; + virtual bool OnInit() Z7_override; + virtual bool OnButtonClicked(unsigned buttonID, HWND buttonHWND) Z7_override; + void EnableSpin(bool enable); + // int AddAction(UINT id); +public: + bool NeedSave; + bool Remember; + bool SkipArc; + bool TestMode; + bool ShowRemember; + // bool ShowSkipFile; + UInt32 Required_GB; + UInt32 Limit_GB; + UString ArcPath; + UString FilePath; + + void AddInfoMessage_To_String(UString &s, UInt64 *ramSize_GB = NULL); + + CMemDialog(): + NeedSave(false), + Remember(false), + SkipArc(false), + TestMode(false), + ShowRemember(true), + // ShowSkipFile(true), + Required_GB(4), + Limit_GB(4) + {} + INT_PTR Create(HWND parentWindow = NULL) { return CModalDialog::Create(IDD_MEM, parentWindow); } +}; + +#endif diff --git a/CPP/7zip/UI/FileManager/MemDialog.rc b/CPP/7zip/UI/FileManager/MemDialog.rc new file mode 100644 index 0000000..d8bcb54 --- /dev/null +++ b/CPP/7zip/UI/FileManager/MemDialog.rc @@ -0,0 +1,49 @@ +#include "MemDialogRes.h" +#include "../../GuiCommon.rc" + +#define xc 320 +#define yc 200 + +#define spin_x_size 50 +#define info_y_size 72 +#define save_y (m + info_y_size + 4) +#define spin_y (save_y + 16) + +#define xsg (xc - m - m) +#define xg (m + m) +#define yg (spin_y + 20) + +IDD_MEM DIALOG MY_MODAL_DIALOG_POSTFIX +CAPTION "Memory usage request" +BEGIN + LTEXT "", IDT_MEM_MESSAGE, m, m, xc, info_y_size, SS_NOPREFIX + CONTROL "Change allowed limit for next operations", IDX_MEM_SAVE_LIMIT, MY_CHECKBOX, + m, save_y, xc, 10 + + MY_CONTROL_EDIT_WITH_SPIN( + IDE_MEM_SPIN_EDIT, + IDC_MEM_SPIN, + "4", m + 10, spin_y, spin_x_size) + + LTEXT "GB", IDT_MEM_GB, m + 10 + spin_x_size + 8, spin_y + 2, 160, 10 + + GROUPBOX "Action", IDG_MEM_ACTION, m, yg, xc, 62 + + MY_CONTROL_AUTORADIOBUTTON_GROUP ( + "&Allow archive unpacking", IDR_MEM_ACTION_ALLOW, + xg, yg + 14, xsg) + MY_CONTROL_AUTORADIOBUTTON ( + "&Skip archive unpacking", IDR_MEM_ACTION_SKIP_ARC, + xg, yg + 28, xsg) +// CONTROL "&Skip file extracting", IDR_MEM_SKIP_FILE, MY_AUTORADIOBUTTON, +// xg, yg + 42, xsg, 10 + + CONTROL "&Repeat selected action for current operation", IDX_MEM_REMEMBER, MY_CHECKBOX, + xg + 10, yg + 44, xsg - 10, 10 + + CONTINUE_CANCEL +END + +#undef save_y +#undef spin_y +#undef spin_x_size diff --git a/CPP/7zip/UI/FileManager/MemDialogRes.h b/CPP/7zip/UI/FileManager/MemDialogRes.h new file mode 100644 index 0000000..9ece82d --- /dev/null +++ b/CPP/7zip/UI/FileManager/MemDialogRes.h @@ -0,0 +1,13 @@ +#define IDD_MEM 7800 + +#define IDX_MEM_SAVE_LIMIT 7801 +#define IDX_MEM_REMEMBER 7802 +#define IDG_MEM_ACTION 7803 + +#define IDR_MEM_ACTION_ALLOW 7820 +#define IDR_MEM_ACTION_SKIP_ARC 7821 + +#define IDT_MEM_MESSAGE 101 +#define IDE_MEM_SPIN_EDIT 110 +#define IDC_MEM_SPIN 111 +#define IDT_MEM_GB 112 diff --git a/CPP/7zip/UI/FileManager/MenuPage.cpp b/CPP/7zip/UI/FileManager/MenuPage.cpp index 2da7f3a..e8736b8 100644 --- a/CPP/7zip/UI/FileManager/MenuPage.cpp +++ b/CPP/7zip/UI/FileManager/MenuPage.cpp @@ -123,9 +123,9 @@ bool CMenuPage::OnInit() bit64.Replace(L"64", L"32"); #endif s.Add_Space(); - s += '('; + s.Add_Char('('); s += bit64; - s += ')'; + s.Add_Char(')'); SetItemText(IDX_SYSTEM_INTEGRATE_TO_MENU_2, s); } diff --git a/CPP/7zip/UI/FileManager/MenuPage.rc b/CPP/7zip/UI/FileManager/MenuPage.rc index fc21107..66162f6 100644 --- a/CPP/7zip/UI/FileManager/MenuPage.rc +++ b/CPP/7zip/UI/FileManager/MenuPage.rc @@ -1,8 +1,8 @@ #include "MenuPageRes.h" #include "../../GuiCommon.rc" -#define xc 240 -#define yc 252 +#define xc OPTIONS_PAGE_XC_SIZE +#define yc OPTIONS_PAGE_YC_SIZE IDD_MENU MY_PAGE #include "MenuPage2.rc" diff --git a/CPP/7zip/UI/FileManager/MenuPage2.rc b/CPP/7zip/UI/FileManager/MenuPage2.rc index 4d1ba21..df0cc54 100644 --- a/CPP/7zip/UI/FileManager/MenuPage2.rc +++ b/CPP/7zip/UI/FileManager/MenuPage2.rc @@ -2,15 +2,15 @@ #define y 96 -#define zoneX 90 +#define zoneX 100 CAPTION "7-Zip" BEGIN - CONTROL "Integrate 7-Zip to shell context menu", IDX_SYSTEM_INTEGRATE_TO_MENU, MY_CHECKBOX, m, m, xc, 10 - CONTROL "(32-bit)", IDX_SYSTEM_INTEGRATE_TO_MENU_2, MY_CHECKBOX, m, m + 14, xc, 10 - CONTROL "Cascaded context menu", IDX_SYSTEM_CASCADED_MENU, MY_CHECKBOX, m, m + 28, xc, 10 - CONTROL "Icons in context menu", IDX_SYSTEM_ICON_IN_MENU, MY_CHECKBOX, m, m + 42, xc, 10 - CONTROL "Eliminate duplication of root folder", IDX_EXTRACT_ELIM_DUP, MY_CHECKBOX, m, m + 56, xc, 10 + MY_CONTROL_CHECKBOX ( "Integrate 7-Zip to shell context menu", IDX_SYSTEM_INTEGRATE_TO_MENU, m, m, xc) + MY_CONTROL_CHECKBOX ( "(32-bit)", IDX_SYSTEM_INTEGRATE_TO_MENU_2, m, m + 14, xc) + MY_CONTROL_CHECKBOX ( "Cascaded context menu", IDX_SYSTEM_CASCADED_MENU, m, m + 28, xc) + MY_CONTROL_CHECKBOX ( "Icons in context menu", IDX_SYSTEM_ICON_IN_MENU, m, m + 42, xc) + MY_CONTROL_CHECKBOX ( "Eliminate duplication of root folder", IDX_EXTRACT_ELIM_DUP, m, m + 56, xc) LTEXT "Propagate Zone.Id stream:", IDT_SYSTEM_ZONE, m, m + 70, xc - zoneX, 8 COMBOBOX IDC_SYSTEM_ZONE, m + xc - zoneX, m + 70 - 2, zoneX, 50, MY_COMBO diff --git a/CPP/7zip/UI/FileManager/MyCom2.h b/CPP/7zip/UI/FileManager/MyCom2.h index d3b49fe..a0cbd45 100644 --- a/CPP/7zip/UI/FileManager/MyCom2.h +++ b/CPP/7zip/UI/FileManager/MyCom2.h @@ -5,20 +5,11 @@ #include "../../../Common/MyCom.h" -#define Z7_COM_ADDREF_RELEASE_MT \ - private: \ - STDMETHOD_(ULONG, AddRef)() Z7_override Z7_final \ - { return (ULONG)InterlockedIncrement((LONG *)&_m_RefCount); } \ - STDMETHOD_(ULONG, Release)() Z7_override Z7_final \ - { const LONG v = InterlockedDecrement((LONG *)&_m_RefCount); \ - if (v != 0) return (ULONG)v; \ - delete this; return 0; } - #define Z7_COM_UNKNOWN_IMP_SPEC_MT2(i1, i) \ Z7_COM_QI_BEGIN \ Z7_COM_QI_ENTRY_UNKNOWN(i1) \ i \ - Z7_COM_QI_END \ + Z7_COM_QI_END_MT \ Z7_COM_ADDREF_RELEASE_MT diff --git a/CPP/7zip/UI/FileManager/MyLoadMenu.cpp b/CPP/7zip/UI/FileManager/MyLoadMenu.cpp index 9453536..51b8648 100644 --- a/CPP/7zip/UI/FileManager/MyLoadMenu.cpp +++ b/CPP/7zip/UI/FileManager/MyLoadMenu.cpp @@ -2,6 +2,7 @@ #include "StdAfx.h" +#include "../../../Windows/FileDir.h" #include "../../../Windows/Menu.h" #include "../../../Windows/TimeUtils.h" #include "../../../Windows/Control/Dialog.h" @@ -12,6 +13,7 @@ #include "AboutDialog.h" #include "App.h" +#include "BrowseDialog2.h" #include "HelpUtils.h" #include "LangUtils.h" #include "MyLoadMenu.h" @@ -27,6 +29,10 @@ static const UINT k_MenuID_SetBookmark = 810; static const UINT k_MenuID_TimePopup = IDM_VIEW_TIME_POPUP; static const UINT k_MenuID_Time = IDM_VIEW_TIME; +#if 0 +// static const UINT k_MenuID_Bookmark_Temp = 850; +#endif + extern HINSTANCE g_hInstance; #define kFMHelpTopic "FM/index.htm" @@ -493,6 +499,9 @@ void OnMenuActivating(HWND /* hWnd */, HMENU hMenu, int position) } if (selectedCommand != 0) menu.CheckRadioItem(k_MenuID_Time, last, selectedCommand, MF_BYCOMMAND); + + if (subMenu.AppendItem(MF_STRING, IDM_VIEW_TIME_UTC, L"UTC")) + subMenu.CheckItemByID(IDM_VIEW_TIME_UTC, g_Timestamp_Show_UTC); } } } @@ -511,9 +520,9 @@ void OnMenuActivating(HWND /* hWnd */, HMENU hMenu, int position) UString s = LangString(IDS_BOOKMARK); s.Add_Space(); const char c = (char)(L'0' + i); - s += c; + s.Add_Char(c); s += "\tAlt+Shift+"; - s += c; + s.Add_Char(c); subMenu.AppendItem(MF_STRING, k_MenuID_SetBookmark + i, s); } @@ -532,9 +541,22 @@ void OnMenuActivating(HWND /* hWnd */, HMENU hMenu, int position) if (s.IsEmpty()) s = '-'; s += "\tAlt+"; - s += (char)('0' + i); + s.Add_Char((char)('0' + i)); menu.AppendItem(MF_STRING, k_MenuID_OpenBookmark + i, s); } +#if 0 + { + FString tempPathF; + if (NFile::NDir::MyGetTempPath(tempPathF)) + { + menu.AppendItem(MF_SEPARATOR, 0, (LPCTSTR)NULL); + UString s; + s = "Temp : "; + s += fs2us(tempPathF); + menu.AppendItem(MF_STRING, k_MenuID_Bookmark_Temp, s); + } + } +#endif } } @@ -741,8 +763,10 @@ bool ExecuteFileCommand(unsigned id) case IDM_HASH_ALL: g_App.CalculateCrc("*"); break; case IDM_CRC32: g_App.CalculateCrc("CRC32"); break; case IDM_CRC64: g_App.CalculateCrc("CRC64"); break; + case IDM_XXH64: g_App.CalculateCrc("XXH64"); break; case IDM_SHA1: g_App.CalculateCrc("SHA1"); break; case IDM_SHA256: g_App.CalculateCrc("SHA256"); break; + case IDM_BLAKE2SP: g_App.CalculateCrc("BLAKE2sp"); break; case IDM_DIFF: g_App.DiffFiles(); break; @@ -878,6 +902,11 @@ bool OnMenuCommand(HWND hWnd, unsigned id) case IDM_VIEW_TOOLBARS_SHOW_BUTTONS_TEXT: g_App.SwitchButtonsLables(); break; case IDM_VIEW_TOOLBARS_LARGE_BUTTONS: g_App.SwitchLargeButtons(); break; + case IDM_VIEW_TIME_UTC: + g_Timestamp_Show_UTC = !g_Timestamp_Show_UTC; + g_App.RedrawListItems_InPanels(); + break; + // Tools case IDM_OPTIONS: OptionsDialog(hWnd, g_hInstance); break; @@ -894,6 +923,19 @@ bool OnMenuCommand(HWND hWnd, unsigned id) dialog.Create(hWnd); break; } + + case IDM_TEMP_DIR: + { + /* + CPanel &panel = g_App.GetFocusedPanel(); + FString tempPathF; + if (NFile::NDir::MyGetTempPath(tempPathF)) + panel.BindToPathAndRefresh(tempPathF); + */ + MyBrowseForTempFolder(g_HWND); + break; + } + default: { if (id >= k_MenuID_OpenBookmark && id <= k_MenuID_OpenBookmark + 9) diff --git a/CPP/7zip/UI/FileManager/OverwriteDialog.cpp b/CPP/7zip/UI/FileManager/OverwriteDialog.cpp index 096527c..f63277a 100644 --- a/CPP/7zip/UI/FileManager/OverwriteDialog.cpp +++ b/CPP/7zip/UI/FileManager/OverwriteDialog.cpp @@ -74,7 +74,7 @@ void COverwriteDialog::SetFileInfoControl(unsigned textID, unsigned iconID, { AddLangString(s, IDS_PROP_MTIME); s += ": "; - char t[32]; + char t[64]; ConvertUtcFileTimeToString(fileInfo.Time, t); s += t; } diff --git a/CPP/7zip/UI/FileManager/Panel.cpp b/CPP/7zip/UI/FileManager/Panel.cpp index 72b72c6..cdb5ba4 100644 --- a/CPP/7zip/UI/FileManager/Panel.cpp +++ b/CPP/7zip/UI/FileManager/Panel.cpp @@ -48,7 +48,6 @@ static DWORD kStyles[4] = { LVS_ICON, LVS_SMALLICON, LVS_LIST, LVS_REPORT }; // static const int kCreateFolderID = 101; extern HINSTANCE g_hInstance; -extern DWORD g_ComCtl32Version; void CPanel::Release() { @@ -449,9 +448,9 @@ bool CPanel::OnCreate(CREATESTRUCT * /* createStruct */) // {VIEW_NEWFOLDER, kCreateFolderID, TBSTATE_ENABLED, BTNS_BUTTON, 0L, 0}, }; - #ifndef UNDER_CE +#ifdef Z7_USE_DYN_ComCtl32Version if (g_ComCtl32Version >= MAKELONG(71, 4)) - #endif +#endif { icex.dwSize = sizeof(INITCOMMONCONTROLSEX); icex.dwICC = ICC_COOL_CLASSES | ICC_BAR_CLASSES; @@ -982,8 +981,8 @@ static UString GetSubFolderNameForExtract2(const UString &arcPath) int CPanel::FindDir_InOperatedList(const CRecordVector &operatedIndices) const { - const bool *isDirVector = &_isDirVector.Front(); - const UInt32 *indices = &operatedIndices.Front(); + const bool *isDirVector = _isDirVector.ConstData(); + const UInt32 *indices = operatedIndices.ConstData(); const unsigned numItems = operatedIndices.Size(); for (unsigned i = 0; i < numItems; i++) if (isDirVector[indices[i]]) @@ -997,7 +996,7 @@ void CPanel::GetFilePaths(const CRecordVector &operatedIndices, UStringV paths.ClearAndReserve(operatedIndices.Size()); UString path = GetFsPath(); const unsigned prefixLen = path.Len(); - const UInt32 *indices = &operatedIndices.Front(); + const UInt32 *indices = operatedIndices.ConstData(); const unsigned numItems = operatedIndices.Size(); // for (unsigned y = 0; y < 10000; y++, paths.Clear()) for (unsigned i = 0; i < numItems; i++) @@ -1030,7 +1029,7 @@ void CPanel::ExtractArchives() if (indices.Size() == 1) outFolder += GetSubFolderNameForExtract2(GetItemRelPath(indices[0])); else - outFolder += '*'; + outFolder.Add_Char('*'); outFolder.Add_PathSepar(); CContextMenuInfo ci; diff --git a/CPP/7zip/UI/FileManager/Panel.h b/CPP/7zip/UI/FileManager/Panel.h index e512cad..5cbc35d 100644 --- a/CPP/7zip/UI/FileManager/Panel.h +++ b/CPP/7zip/UI/FileManager/Panel.h @@ -60,6 +60,11 @@ const UInt32 kParentIndex_UInt32 = (UInt32)(Int32)kParentIndex; #define ROOT_FS_FOLDER L"C:\\" #endif +#if !defined(Z7_WIN32_WINNT_MIN) || Z7_WIN32_WINNT_MIN < 0x0500 // < win2000 +#define Z7_USE_DYN_ComCtl32Version +extern DWORD g_ComCtl32Version; +#endif + Z7_PURE_INTERFACES_BEGIN DECLARE_INTERFACE(CPanelCallback) diff --git a/CPP/7zip/UI/FileManager/PanelCopy.cpp b/CPP/7zip/UI/FileManager/PanelCopy.cpp index de3d764..40a347f 100644 --- a/CPP/7zip/UI/FileManager/PanelCopy.cpp +++ b/CPP/7zip/UI/FileManager/PanelCopy.cpp @@ -102,7 +102,7 @@ HRESULT CPanelCopyThread::ProcessVirt() NExtract::NPathMode::EEnum pathMode = NExtract::NPathMode::kCurPaths; // NExtract::NPathMode::kFullPathnames; - result2 = archiveFolder->Extract(&Indices.Front(), Indices.Size(), + result2 = archiveFolder->Extract(Indices.ConstData(), Indices.Size(), BoolToInt(options->includeAltStreams), BoolToInt(options->replaceAltStreamChars), pathMode, NExtract::NOverwriteMode::kAsk, @@ -111,7 +111,7 @@ HRESULT CPanelCopyThread::ProcessVirt() else result2 = FolderOperations->CopyTo( BoolToInt(options->moveMode), - &Indices.Front(), Indices.Size(), + Indices.ConstData(), Indices.Size(), BoolToInt(options->includeAltStreams), BoolToInt(options->replaceAltStreamChars), options->folder, ExtractCallback); @@ -316,7 +316,7 @@ struct CThreadUpdate Result = FolderOperations->CopyFrom( MoveMode, FolderPrefix, - &FileNamePointers.Front(), + FileNamePointers.ConstData(), FileNamePointers.Size(), UpdateCallback); } diff --git a/CPP/7zip/UI/FileManager/PanelFolderChange.cpp b/CPP/7zip/UI/FileManager/PanelFolderChange.cpp index a13b88d..406e304 100644 --- a/CPP/7zip/UI/FileManager/PanelFolderChange.cpp +++ b/CPP/7zip/UI/FileManager/PanelFolderChange.cpp @@ -907,7 +907,7 @@ void CPanel::OpenAltStreams() path.DeleteBack(); } - path += ':'; + path.Add_Colon(); BindToPathAndRefresh(path); #endif } diff --git a/CPP/7zip/UI/FileManager/PanelItemOpen.cpp b/CPP/7zip/UI/FileManager/PanelItemOpen.cpp index bd65aef..f4ee3f3 100644 --- a/CPP/7zip/UI/FileManager/PanelItemOpen.cpp +++ b/CPP/7zip/UI/FileManager/PanelItemOpen.cpp @@ -9,7 +9,6 @@ #include "../../../Common/IntToString.h" #include "../../../Common/AutoPtr.h" -#include "../../../Common/StringConvert.h" #include "../../../Windows/ProcessUtils.h" #include "../../../Windows/FileName.h" @@ -214,6 +213,8 @@ static void My_GetProcessFileName_2(HANDLE hProcess, UString &path) } */ +Z7_DIAGNOSTIC_IGNORE_CAST_FUNCTION + static void My_GetProcessFileName(HANDLE hProcess, UString &path) { path.Empty(); @@ -626,14 +627,14 @@ HRESULT CPanel::OpenParentArchiveFolder() static const char * const kExeExtensions = - " exe bat ps1 com" + " exe bat ps1 com lnk" " "; static const char * const kStartExtensions = #ifdef UNDER_CE " cab" #endif - " exe bat ps1 com" + " exe bat ps1 com lnk" " chm" " msi doc dot xls ppt pps wps wpt wks xlr wdb vsd pub" @@ -813,104 +814,8 @@ void CApp::DiffFiles(const UString &path1, const UString &path2) } -#ifndef _UNICODE -typedef BOOL (WINAPI * Func_ShellExecuteExW)(LPSHELLEXECUTEINFOW lpExecInfo); -#endif - -static HRESULT StartApplication(const UString &dir, const UString &path, HWND window, CProcess &process) -{ - UString path2 = path; - - #ifdef _WIN32 - { - int dot = path2.ReverseFind_Dot(); - int separ = path2.ReverseFind_PathSepar(); - if (dot < 0 || dot < separ) - path2.Add_Dot(); - } - #endif - - UINT32 result; - - #ifndef _UNICODE - if (g_IsNT) - { - SHELLEXECUTEINFOW execInfo; - execInfo.cbSize = sizeof(execInfo); - execInfo.fMask = SEE_MASK_NOCLOSEPROCESS | SEE_MASK_FLAG_DDEWAIT; - execInfo.hwnd = NULL; - execInfo.lpVerb = NULL; - execInfo.lpFile = path2; - execInfo.lpParameters = NULL; - execInfo.lpDirectory = dir.IsEmpty() ? NULL : (LPCWSTR)dir; - execInfo.nShow = SW_SHOWNORMAL; - execInfo.hProcess = NULL; - const - Func_ShellExecuteExW - f_ShellExecuteExW = Z7_GET_PROC_ADDRESS( - Func_ShellExecuteExW, ::GetModuleHandleW(L"shell32.dll"), - "ShellExecuteExW"); - if (!f_ShellExecuteExW) - return 0; - f_ShellExecuteExW(&execInfo); - result = (UINT32)(UINT_PTR)execInfo.hInstApp; - process.Attach(execInfo.hProcess); - } - else - #endif - { - SHELLEXECUTEINFO execInfo; - execInfo.cbSize = sizeof(execInfo); - execInfo.fMask = SEE_MASK_NOCLOSEPROCESS - #ifndef UNDER_CE - | SEE_MASK_FLAG_DDEWAIT - #endif - ; - execInfo.hwnd = NULL; - execInfo.lpVerb = NULL; - const CSysString sysPath (GetSystemString(path2)); - const CSysString sysDir (GetSystemString(dir)); - execInfo.lpFile = sysPath; - execInfo.lpParameters = NULL; - execInfo.lpDirectory = - #ifdef UNDER_CE - NULL - #else - sysDir.IsEmpty() ? NULL : (LPCTSTR)sysDir - #endif - ; - execInfo.nShow = SW_SHOWNORMAL; - execInfo.hProcess = NULL; - ::ShellExecuteEx(&execInfo); - result = (UINT32)(UINT_PTR)execInfo.hInstApp; - process.Attach(execInfo.hProcess); - } - - - DEBUG_PRINT_NUM("-- ShellExecuteEx -- execInfo.hInstApp = ", result) - - if (result <= 32) - { - switch (result) - { - case SE_ERR_NOASSOC: - ::MessageBoxW(window, - NError::MyFormatMessage(::GetLastError()), - // L"There is no application associated with the given file name extension", - L"7-Zip", MB_OK | MB_ICONSTOP); - } - - return E_FAIL; // fixed in 15.13. Can we use it for any Windows version? - } - - return S_OK; -} - -static void StartApplicationDontWait(const UString &dir, const UString &path, HWND window) -{ - CProcess process; - StartApplication(dir, path, window, process); -} +HRESULT StartApplication(const UString &dir, const UString &path, HWND window, CProcess &process); +void StartApplicationDontWait(const UString &dir, const UString &path, HWND window); void CPanel::EditItem(unsigned index, bool useEditor) { @@ -1199,8 +1104,7 @@ static THREAD_FUNC_DECL MyThreadFunction(void *param) { DEBUG_PRINT("==== MyThreadFunction ===="); - CMyAutoPtr tmpProcessInfoPtr((CTmpProcessInfo *)param); - CTmpProcessInfo *tpi = tmpProcessInfoPtr.get(); + CMyUniquePtr tpi((CTmpProcessInfo *)param); CChildProcesses &processes = tpi->Processes; bool mainProcessWasSet = !processes.Handles.IsEmpty(); @@ -1257,7 +1161,7 @@ static THREAD_FUNC_DECL MyThreadFunction(void *param) { handles.Add(g_ExitEventLauncher._exitEvent); - DWORD waitResult = WaitForMultiObj_Any_Infinite(handles.Size(), &handles.Front()); + DWORD waitResult = WaitForMultiObj_Any_Infinite(handles.Size(), handles.ConstData()); waitResult -= WAIT_OBJECT_0; @@ -1348,7 +1252,7 @@ static THREAD_FUNC_DECL MyThreadFunction(void *param) if (::MessageBoxW(g_HWND, message, L"7-Zip", MB_OKCANCEL | MB_ICONQUESTION) == IDOK) { // DEBUG_PRINT_NUM("SendMessage", GetCurrentThreadId()); - if (SendMessage(tpi->Window, kOpenItemChanged, 0, (LONG_PTR)tpi) != 1) + if (SendMessage(tpi->Window, kOpenItemChanged, 0, (LONG_PTR)tpi.get()) != 1) { ::MessageBoxW(g_HWND, m, L"7-Zip", MB_OK | MB_ICONSTOP); return 0; @@ -1686,6 +1590,19 @@ void CPanel::OpenItemInArchive(unsigned index, bool tryInternal, bool tryExterna options.streamMode = true; virtFileSystemSpec = new CVirtFileSystem; virtFileSystem = virtFileSystemSpec; + +#if defined(_WIN32) && !defined(UNDER_CE) +#ifndef _UNICODE + if (g_IsNT) +#endif + if (_parentFolders.Size() > 0) + { + const CFolderLink &fl = _parentFolders.Front(); + if (!fl.IsVirtual && !fl.FilePath.IsEmpty()) + ReadZoneFile_Of_BaseFile(fl.FilePath, virtFileSystemSpec->ZoneBuf); + } +#endif + // we allow additional total size for small alt streams; virtFileSystemSpec->MaxTotalAllocSize = fileSize + (1 << 10); @@ -1784,8 +1701,7 @@ void CPanel::OpenItemInArchive(unsigned index, bool tryInternal, bool tryExterna if (!tryExternal) return; - CMyAutoPtr tmpProcessInfoPtr(new CTmpProcessInfo()); - CTmpProcessInfo *tpi = tmpProcessInfoPtr.get(); + CMyUniquePtr tpi(new CTmpProcessInfo()); tpi->FolderPath = tempDir; tpi->FilePath = tempFilePath; tpi->NeedDelete = true; @@ -1825,13 +1741,13 @@ void CPanel::OpenItemInArchive(unsigned index, bool tryInternal, bool tryExterna tpi->Processes.SetMainProcess(process.Detach()); ::CThread th; - if (Thread_Create(&th, MyThreadFunction, tpi) != 0) + if (Thread_Create(&th, MyThreadFunction, tpi.get()) != 0) throw 271824; g_ExitEventLauncher._threads.Add(th); g_ExitEventLauncher._numActiveThreads++; tempDirectory.DisableDeleting(); - tmpProcessInfoPtr.release(); + tpi.release(); tmpProcessInfoRelease._needDelete = false; } diff --git a/CPP/7zip/UI/FileManager/PanelItems.cpp b/CPP/7zip/UI/FileManager/PanelItems.cpp index 0cb33d8..2335fc0 100644 --- a/CPP/7zip/UI/FileManager/PanelItems.cpp +++ b/CPP/7zip/UI/FileManager/PanelItems.cpp @@ -89,7 +89,8 @@ static int GetColumnAlign(PROPID propID, VARTYPE varType) static int ItemProperty_Compare_NameFirst(void *const *a1, void *const *a2, void * /* param */) { - return (*(*((const CPropColumn *const *)a1))).Compare_NameFirst(*(*((const CPropColumn *const *)a2))); + return (*(*((const CPropColumn *const *)a1))).Compare_NameFirst + (*(*((const CPropColumn *const *)a2))); } HRESULT CPanel::InitColumns() @@ -125,7 +126,7 @@ HRESULT CPanel::InitColumns() _columns.Clear(); - bool isFsFolder = IsFSFolder() || IsAltStreamsFolder(); + const bool isFsFolder = IsFSFolder() || IsAltStreamsFolder(); { UInt32 numProps; @@ -182,7 +183,7 @@ HRESULT CPanel::InitColumns() { CMyComBSTR name; PROPID propID; - HRESULT res = _folderRawProps->GetRawPropInfo(i, &name, &propID); + const HRESULT res = _folderRawProps->GetRawPropInfo(i, &name, &propID); if (res != S_OK) continue; CPropColumn prop; @@ -550,7 +551,7 @@ HRESULT CPanel::RefreshListCtrl(const CSelectedState &state) { NCOM::CPropVariant prop; _isDirVector.ClearAndSetSize(numItems); - bool *vec = (bool *)&_isDirVector.Front(); + bool *vec = _isDirVector.NonConstData(); HRESULT hres = S_OK; unsigned i; for (i = 0; i < numItems; i++) @@ -892,7 +893,7 @@ void CPanel::Get_ItemIndices_Selected(CRecordVector &indices) const } HeapSort(&indices.Front(), indices.Size()); */ - const bool *v = &_selectedStatusVector.Front(); + const bool *v = _selectedStatusVector.ConstData(); const unsigned size = _selectedStatusVector.Size(); for (unsigned i = 0; i < size; i++) if (v[i]) @@ -926,7 +927,7 @@ void CPanel::Get_ItemIndices_All(CRecordVector &indices) const if (_folder->GetNumberOfItems(&numItems) != S_OK) return; indices.ClearAndSetSize(numItems); - UInt32 *vec = (UInt32 *)&indices.Front(); + UInt32 *vec = indices.NonConstData(); for (UInt32 i = 0; i < numItems; i++) vec[i] = i; } @@ -1258,7 +1259,7 @@ void CPanel::SaveListViewInfo() CListViewInfo viewInfo; // PROPID sortPropID = _columns[_sortIndex].ID; - PROPID sortPropID = _sortID; + const PROPID sortPropID = _sortID; // we save columns as "sorted by order" to registry @@ -1300,9 +1301,9 @@ void CPanel::SaveListViewInfo() } -bool CPanel::OnRightClick(MY_NMLISTVIEW_NMITEMACTIVATE *itemActiveate, LRESULT &result) +bool CPanel::OnRightClick(MY_NMLISTVIEW_NMITEMACTIVATE *itemActivate, LRESULT &result) { - if (itemActiveate->hdr.hwndFrom == HWND(_listView)) + if (itemActivate->hdr.hwndFrom == HWND(_listView)) return false; POINT point; ::GetCursorPos(&point); diff --git a/CPP/7zip/UI/FileManager/PanelListNotify.cpp b/CPP/7zip/UI/FileManager/PanelListNotify.cpp index 2fb0e87..4dbd9f6 100644 --- a/CPP/7zip/UI/FileManager/PanelListNotify.cpp +++ b/CPP/7zip/UI/FileManager/PanelListNotify.cpp @@ -32,12 +32,12 @@ using namespace NWindows; #define SPACE_TERMINATOR_CHAR (wchar_t)(0x9C) #define INT_TO_STR_SPEC(v) \ - while (v >= 10) { temp[i++] = (unsigned char)('0' + (unsigned)(v % 10)); v /= 10; } \ - *s++ = (unsigned char)('0' + (unsigned)v); + while (v >= 10) { temp[i++] = (Byte)('0' + (unsigned)(v % 10)); v /= 10; } \ + *s++ = (Byte)('0' + (unsigned)v); static void ConvertSizeToString(UInt64 val, wchar_t *s) throw() { - unsigned char temp[32]; + Byte temp[32]; unsigned i = 0; if (val <= (UInt32)0xFFFFFFFF) @@ -92,30 +92,6 @@ UString ConvertSizeToString(UInt64 value) return s; } -static inline unsigned GetHex_Upper(unsigned v) -{ - return (v < 10) ? ('0' + v) : ('A' + (v - 10)); -} - -static inline unsigned GetHex_Lower(unsigned v) -{ - return (v < 10) ? ('0' + v) : ('a' + (v - 10)); -} - -/* -static void HexToString(char *dest, const Byte *data, UInt32 size) -{ - for (UInt32 i = 0; i < size; i++) - { - unsigned b = data[i]; - dest[0] = GetHex((b >> 4) & 0xF); - dest[1] = GetHex(b & 0xF); - dest += 2; - } - *dest = 0; -} -*/ - bool IsSizeProp(UINT propID) throw(); bool IsSizeProp(UINT propID) throw() { @@ -220,6 +196,8 @@ LRESULT CPanel::SetItemText(LVITEMW &item) if (item.cchTextMax <= 1) return 0; + + // item.cchTextMax > 1 const CPropColumn &property = _visibleColumns[item.iSubItem]; PROPID propID = property.ID; @@ -290,12 +268,10 @@ LRESULT CPanel::SetItemText(LVITEMW &item) UInt32 dataSize; UInt32 propType; RINOK(_folderRawProps->GetRawProp(realIndex, propID, &data, &dataSize, &propType)) - const unsigned limit = (unsigned)item.cchTextMax - 1; + unsigned limit = (unsigned)item.cchTextMax - 1; + // limit != 0 if (dataSize == 0) - { - text[0] = 0; return 0; - } if (propID == kpidNtReparse) { @@ -306,7 +282,7 @@ LRESULT CPanel::SetItemText(LVITEMW &item) unsigned i; for (i = 0; i < limit; i++) { - wchar_t c = s[i]; + const wchar_t c = s[i]; if (c == 0) break; text[i] = c; @@ -324,7 +300,7 @@ LRESULT CPanel::SetItemText(LVITEMW &item) unsigned i; for (i = 0; i < limit; i++) { - wchar_t c = (Byte)s[i]; + const wchar_t c = (Byte)s[i]; if (c == 0) break; text[i] = c; @@ -346,33 +322,29 @@ LRESULT CPanel::SetItemText(LVITEMW &item) wchar_t c = (Byte)temp[i]; if (c == 0) break; - text[i] = c; + *text++ = c; } - text[i] = 0; + *text = 0; } else { - if (dataSize > limit) - dataSize = limit; - WCHAR *dest = text; - const bool needUpper = (dataSize <= 8) - && (propID == kpidCRC || propID == kpidChecksum); - for (UInt32 i = 0; i < dataSize; i++) + const char * const k_Hex = + (dataSize <= 8 + && (propID == kpidCRC || propID == kpidChecksum)) + ? k_Hex_Upper : k_Hex_Lower; + limit /= 2; + if (limit > dataSize) + limit = dataSize; + const Byte *data2 = (const Byte *)data; + do { - unsigned b = ((const Byte *)data)[i]; - if (needUpper) - { - dest[0] = (WCHAR)GetHex_Upper((b >> 4) & 0xF); - dest[1] = (WCHAR)GetHex_Upper(b & 0xF); - } - else - { - dest[0] = (WCHAR)GetHex_Lower((b >> 4) & 0xF); - dest[1] = (WCHAR)GetHex_Lower(b & 0xF); - } - dest += 2; + const size_t b = *data2++; + text[0] = (Byte)k_Hex[b >> 4]; + text[1] = (Byte)k_Hex[b & 15]; + text += 2; } - *dest = 0; + while (--limit); + *text = 0; } } return 0; @@ -549,10 +521,6 @@ LRESULT CPanel::SetItemText(LVITEMW &item) return 0; } -#ifndef UNDER_CE -extern DWORD g_ComCtl32Version; -#endif - void CPanel::OnItemChanged(NMLISTVIEW *item) { const unsigned index = (unsigned)item->lParam; @@ -691,9 +659,9 @@ bool CPanel::OnNotifyList(LPNMHDR header, LRESULT &result) SetFocusToList(); Post_Refresh_StatusBar(); if (_mySelectMode) - #ifndef UNDER_CE +#ifdef Z7_USE_DYN_ComCtl32Version if (g_ComCtl32Version >= MAKELONG(71, 4)) - #endif +#endif OnLeftClick((MY_NMLISTVIEW_NMITEMACTIVATE *)header); return false; } @@ -843,7 +811,7 @@ void CPanel::Refresh_StatusBar() NCOM::CPropVariant prop; if (_folder->GetProperty(realIndex, kpidMTime, &prop) == S_OK) { - char dateString2[32]; + char dateString2[64]; dateString2[0] = 0; ConvertPropertyToShortString2(dateString2, prop, kpidMTime); for (unsigned i = 0;; i++) diff --git a/CPP/7zip/UI/FileManager/PanelMenu.cpp b/CPP/7zip/UI/FileManager/PanelMenu.cpp index f0afe15..9086996 100644 --- a/CPP/7zip/UI/FileManager/PanelMenu.cpp +++ b/CPP/7zip/UI/FileManager/PanelMenu.cpp @@ -155,16 +155,6 @@ static void AddPropertyString(PROPID propID, UInt64 val, CListViewDialog &dialog } -static inline unsigned GetHex_Upper(unsigned v) -{ - return (v < 10) ? ('0' + v) : ('A' + (v - 10)); -} - -static inline unsigned GetHex_Lower(unsigned v) -{ - return (v < 10) ? ('0' + v) : ('a' + (v - 10)); -} - static const Byte kSpecProps[] = { kpidPath, @@ -244,7 +234,7 @@ void CPanel::Properties() ConvertNtSecureToString((const Byte *)data, dataSize, s); else { - const UInt32 kMaxDataSize = 64; + const unsigned kMaxDataSize = 1 << 8; if (dataSize > kMaxDataSize) { s += "data:"; @@ -252,22 +242,12 @@ void CPanel::Properties() } else { - const bool needUpper = (dataSize <= 8) - && (propID == kpidCRC || propID == kpidChecksum); - for (UInt32 k = 0; k < dataSize; k++) - { - const Byte b = ((const Byte *)data)[k]; - if (needUpper) - { - s += (char)GetHex_Upper((b >> 4) & 0xF); - s += (char)GetHex_Upper(b & 0xF); - } - else - { - s += (char)GetHex_Lower((b >> 4) & 0xF); - s += (char)GetHex_Lower(b & 0xF); - } - } + char temp[kMaxDataSize * 2 + 2]; + if (dataSize <= 8 && (propID == kpidCRC || propID == kpidChecksum)) + ConvertDataToHex_Upper(temp, (const Byte *)data, dataSize); + else + ConvertDataToHex_Lower(temp, (const Byte *)data, dataSize); + s += temp; } } AddPropertyPair(GetNameOfProperty(propID, name), (UString)s.Ptr(), message); @@ -622,7 +602,7 @@ HRESULT CPanel::CreateShellContextMenu( ODS("==== CPanel::CreateShellContextMenu pidls END"); // Get IContextMenu for items RINOK(parentFolder->GetUIObjectOf(GetParent(), - pidls.items.Size(), (LPCITEMIDLIST *)(void *)&pidls.items.Front(), + pidls.items.Size(), (LPCITEMIDLIST *)(void *)pidls.items.ConstData(), IID_IContextMenu, NULL, (void**)&systemContextMenu)) ODS("==== CPanel::CreateShellContextMenu GetUIObjectOf finished"); if (!systemContextMenu) diff --git a/CPP/7zip/UI/FileManager/PanelOperations.cpp b/CPP/7zip/UI/FileManager/PanelOperations.cpp index b61f4e9..6c2cea1 100644 --- a/CPP/7zip/UI/FileManager/PanelOperations.cpp +++ b/CPP/7zip/UI/FileManager/PanelOperations.cpp @@ -60,7 +60,7 @@ HRESULT CThreadFolderOperations::ProcessVirt() case FOLDER_TYPE_CREATE_FOLDER: return FolderOperations->CreateFolder(Name, UpdateCallback); case FOLDER_TYPE_DELETE: - return FolderOperations->Delete(&Indices.Front(), Indices.Size(), UpdateCallback); + return FolderOperations->Delete(Indices.ConstData(), Indices.Size(), UpdateCallback); case FOLDER_TYPE_RENAME: return FolderOperations->Rename(Index, Name, UpdateCallback); default: @@ -192,6 +192,8 @@ void CPanel::DeleteItems(bool NON_CE_VAR(toRecycleBin)) #ifdef _UNICODE /* res = */ ::SHFileOperationW(&fo); #else +Z7_DIAGNOSTIC_IGNORE_CAST_FUNCTION + const Func_SHFileOperationW f_SHFileOperationW = Z7_GET_PROC_ADDRESS( Func_SHFileOperationW, ::GetModuleHandleW(L"shell32.dll"), @@ -316,7 +318,7 @@ BOOL CPanel::OnEndLabelEdit(LV_DISPINFOW * lpnmh) if (realIndex == kParentIndex) return FALSE; const UString prefix = GetItemPrefix(realIndex); - + const UString oldName = GetItemName(realIndex); CDisableNotify disableNotify(*this); { @@ -324,20 +326,22 @@ BOOL CPanel::OnEndLabelEdit(LV_DISPINFOW * lpnmh) op.FolderOperations = _folderOperations; op.Index = realIndex; op.Name = newName; - /* HRESULTres = */ op.DoOperation(*this, + const HRESULT res = op.DoOperation(*this, LangString(IDS_RENAMING), LangString(IDS_ERROR_RENAMING)); // fixed in 9.26: we refresh list even after errors // (it's more safe, since error can be at different stages, so list can be incorrect). - /* - if (res != S_OK) - return FALSE; - */ + if (res == S_OK) + _selectedState.FocusedName = prefix + newName; + else + { + _selectedState.FocusedName = prefix + oldName; + // return FALSE; + } } // Can't use RefreshListCtrl here. // RefreshListCtrlSaveFocused(); - _selectedState.FocusedName = prefix + newName; _selectedState.FocusedName_Defined = true; _selectedState.SelectFocused = true; diff --git a/CPP/7zip/UI/FileManager/PanelSplitFile.cpp b/CPP/7zip/UI/FileManager/PanelSplitFile.cpp index 64aa039..c452d41 100644 --- a/CPP/7zip/UI/FileManager/PanelSplitFile.cpp +++ b/CPP/7zip/UI/FileManager/PanelSplitFile.cpp @@ -38,7 +38,7 @@ struct CVolSeqName while (numVolumes > 999) { numVolumes /= 10; - ChangedPart += '0'; + ChangedPart.Add_Char('0'); } } @@ -192,7 +192,7 @@ HRESULT CThreadSplit::ProcessVirt() name.Add_Dot(); name += us2fs(seqName.GetNextName()); sync.Set_FilePath(fs2us(name)); - if (!outFile.File.Create(name, false)) + if (!outFile.File.Create_NEW(name)) { const HRESULT res = GetLastError_noZero_HRESULT(); AddErrorPath(name); @@ -277,7 +277,7 @@ void CApp::Split() srcPanel.MessageBox_Error(L"Cannot find file"); return; } - if (fileInfo.Size <= splitDialog.VolumeSizes.Front()) + if (fileInfo.Size <= splitDialog.VolumeSizes.FrontItem()) { srcPanel.MessageBox_Error_LangID(IDS_SPLIT_VOL_MUST_BE_SMALLER); return; @@ -355,7 +355,7 @@ class CThreadCombine: public CProgressThreadVirt HRESULT CThreadCombine::ProcessVirt() { NIO::COutFile outFile; - if (!outFile.Create(OutputPath, false)) + if (!outFile.Create_NEW(OutputPath)) { const HRESULT res = GetLastError_noZero_HRESULT(); AddErrorPath(OutputPath); diff --git a/CPP/7zip/UI/FileManager/PasswordDialog.rc b/CPP/7zip/UI/FileManager/PasswordDialog.rc index 90c57ef..fa42f92 100644 --- a/CPP/7zip/UI/FileManager/PasswordDialog.rc +++ b/CPP/7zip/UI/FileManager/PasswordDialog.rc @@ -1,7 +1,11 @@ #include "PasswordDialogRes.h" #include "../../GuiCommon.rc" +#ifdef UNDER_CE #define xc 140 +#else +#define xc 200 +#endif #define yc 72 IDD_PASSWORD DIALOG 0, 0, xs, ys MY_MODAL_DIALOG_STYLE MY_FONT diff --git a/CPP/7zip/UI/FileManager/PluginLoader.h b/CPP/7zip/UI/FileManager/PluginLoader.h index d9309f2..476a7b7 100644 --- a/CPP/7zip/UI/FileManager/PluginLoader.h +++ b/CPP/7zip/UI/FileManager/PluginLoader.h @@ -7,13 +7,15 @@ #include "IFolder.h" +Z7_DIAGNOSTIC_IGNORE_CAST_FUNCTION + class CPluginLibrary: public NWindows::NDLL::CLibrary { public: HRESULT CreateManager(REFGUID clsID, IFolderManager **manager) { const - Func_CreateObject createObject = Z7_GET_PROC_ADDRESS( + Func_CreateObject createObject = Z7_GET_PROC_ADDRESS( Func_CreateObject, Get_HMODULE(), "CreateObject"); if (!createObject) diff --git a/CPP/7zip/UI/FileManager/ProgressDialog2.cpp b/CPP/7zip/UI/FileManager/ProgressDialog2.cpp index 1521d83..690ebec 100644 --- a/CPP/7zip/UI/FileManager/ProgressDialog2.cpp +++ b/CPP/7zip/UI/FileManager/ProgressDialog2.cpp @@ -23,6 +23,7 @@ using namespace NWindows; extern HINSTANCE g_hInstance; +extern bool g_DisableUserQuestions; static const UINT_PTR kTimerID = 3; @@ -977,6 +978,7 @@ INT_PTR CProgressDialog::Create(const UString &title, NWindows::CThread &thread, } thread.Wait_Close(); if (!MessagesDisplayed) + if (!g_DisableUserQuestions) MessageBoxW(wndParent, L"Progress Error", L"7-Zip", MB_ICONERROR); return res; } @@ -1012,7 +1014,8 @@ bool CProgressDialog::OnExternalCloseMessage() MessagesDisplayed = true; if (fm.ErrorMessage.Title.IsEmpty()) fm.ErrorMessage.Title = "7-Zip"; - MessageBoxW(*this, fm.ErrorMessage.Message, fm.ErrorMessage.Title, MB_ICONERROR); + if (!g_DisableUserQuestions) + MessageBoxW(*this, fm.ErrorMessage.Message, fm.ErrorMessage.Title, MB_ICONERROR); } else if (!thereAreMessages) { @@ -1022,10 +1025,12 @@ bool CProgressDialog::OnExternalCloseMessage() { if (fm.OkMessage.Title.IsEmpty()) fm.OkMessage.Title = "7-Zip"; - MessageBoxW(*this, fm.OkMessage.Message, fm.OkMessage.Title, MB_OK); + if (!g_DisableUserQuestions) + MessageBoxW(*this, fm.OkMessage.Message, fm.OkMessage.Title, MB_OK); } } + if (!g_DisableUserQuestions) if (thereAreMessages && !_cancelWasPressed) { _waitCloseByCancelButton = true; @@ -1084,7 +1089,7 @@ void CProgressDialog::SetTitleText() char temp[32]; ConvertUInt64ToString(_prevPercentValue, temp); s += temp; - s += '%'; + s.Add_Char('%'); } if (!_foreground) { diff --git a/CPP/7zip/UI/FileManager/RegistryAssociations.cpp b/CPP/7zip/UI/FileManager/RegistryAssociations.cpp index 16e4675..c4465da 100644 --- a/CPP/7zip/UI/FileManager/RegistryAssociations.cpp +++ b/CPP/7zip/UI/FileManager/RegistryAssociations.cpp @@ -143,7 +143,7 @@ LONG AddShellExtensionInfo(HKEY hkey, iconIndex = 0; // if (iconIndex >= 0) { - iconPathFull += ','; + iconPathFull.Add_Char(','); iconPathFull.Add_UInt32((UInt32)iconIndex); } iconKey.Create(programKey, kDefaultIconKeyName); diff --git a/CPP/7zip/UI/FileManager/RootFolder.cpp b/CPP/7zip/UI/FileManager/RootFolder.cpp index 34dd638..606fb7f 100644 --- a/CPP/7zip/UI/FileManager/RootFolder.cpp +++ b/CPP/7zip/UI/FileManager/RootFolder.cpp @@ -124,13 +124,20 @@ Z7_COM7F_IMF(CRootFolder::GetProperty(UInt32 itemIndex, PROPID propID, PROPVARIA return S_OK; } +#if !defined(Z7_WIN32_WINNT_MIN) || Z7_WIN32_WINNT_MIN < 0x0400 // nt4 +#define Z7_USE_DYN_SHGetSpecialFolderPath +#endif + +#ifdef Z7_USE_DYN_SHGetSpecialFolderPath typedef BOOL (WINAPI *Func_SHGetSpecialFolderPathW)(HWND hwnd, LPWSTR pszPath, int csidl, BOOL fCreate); typedef BOOL (WINAPI *Func_SHGetSpecialFolderPathA)(HWND hwnd, LPSTR pszPath, int csidl, BOOL fCreate); +#endif static UString GetMyDocsPath() { UString us; WCHAR s[MAX_PATH + 1]; +#ifdef Z7_USE_DYN_SHGetSpecialFolderPath #ifdef UNDER_CE #define shell_name TEXT("coredll.dll") #else @@ -139,16 +146,25 @@ static UString GetMyDocsPath() Func_SHGetSpecialFolderPathW getW = Z7_GET_PROC_ADDRESS( Func_SHGetSpecialFolderPathW, GetModuleHandle(shell_name), "SHGetSpecialFolderPathW"); - if (getW && getW(NULL, s, CSIDL_PERSONAL, FALSE)) + if (getW && getW +#else + if (SHGetSpecialFolderPathW +#endif + (NULL, s, CSIDL_PERSONAL, FALSE)) us = s; #ifndef _UNICODE else { + CHAR s2[MAX_PATH + 1]; +#ifdef Z7_USE_DYN_SHGetSpecialFolderPath Func_SHGetSpecialFolderPathA getA = Z7_GET_PROC_ADDRESS( Func_SHGetSpecialFolderPathA, ::GetModuleHandleA("shell32.dll"), "SHGetSpecialFolderPathA"); - CHAR s2[MAX_PATH + 1]; - if (getA && getA(NULL, s2, CSIDL_PERSONAL, FALSE)) + if (getA && getA +#else + if (SHGetSpecialFolderPathA +#endif + (NULL, s2, CSIDL_PERSONAL, FALSE)) us = GetUnicodeString(s2); } #endif diff --git a/CPP/7zip/UI/FileManager/SettingsPage.cpp b/CPP/7zip/UI/FileManager/SettingsPage.cpp index 784b8c7..a5117be 100644 --- a/CPP/7zip/UI/FileManager/SettingsPage.cpp +++ b/CPP/7zip/UI/FileManager/SettingsPage.cpp @@ -2,15 +2,21 @@ #include "StdAfx.h" +#include + // #include "../../../Common/IntToString.h" // #include "../../../Common/StringConvert.h" +#include "../../../Common/StringToInt.h" #ifndef UNDER_CE +#include "../../../Windows/ErrorMsg.h" #include "../../../Windows/MemoryLock.h" -// #include "../../../Windows/System.h" +#include "../../../Windows/System.h" #endif -// #include "../Common/ZipRegistry.h" +#include "../Explorer/MyMessages.h" + +#include "../Common/ZipRegistry.h" #include "HelpUtils.h" #include "LangUtils.h" @@ -30,7 +36,8 @@ static const UInt32 kLangIDs[] = IDX_SETTINGS_SHOW_GRID, IDX_SETTINGS_SINGLE_CLICK, IDX_SETTINGS_ALTERNATIVE_SELECTION, - IDX_SETTINGS_LARGE_PAGES + IDX_SETTINGS_LARGE_PAGES, + IDT_MEM_USAGE_EXTRACT // , IDT_COMPRESS_MEMORY }; #endif @@ -106,8 +113,10 @@ int CSettingsPage::AddMemComboItem(UInt64 size, UInt64 percents, bool isDefault) bool CSettingsPage::OnInit() { + _initMode = true; _wasChanged = false; _largePages_wasChanged = false; + _memx_wasChanged = false; /* _wasChanged_MemLimit = false; _memLimitStrings.Clear(); @@ -187,9 +196,64 @@ bool CSettingsPage::OnInit() // EnableSubItems(); + + { + UInt64 ramSize = (UInt64)sizeof(size_t) << 29; + const bool ramSize_defined = NWindows::NSystem::GetRamSize(ramSize); + // ramSize *= 10; // for debug + UInt64 ramSize_GB = (ramSize + (1u << 29)) >> 30; + if (ramSize_GB == 0) + ramSize_GB = 1; + UString s ("GB"); + if (ramSize_defined) + { + s += " / "; + s.Add_UInt64(ramSize_GB); + s += " GB (RAM)"; + } + SetItemText(IDT_SETTINGS_MEM_GB, s); + + const UINT valMin = 1; + UINT valMax = 64; // 64GB for RAR7 + if (ramSize_defined /* && ramSize_GB > valMax */) + { + const UINT k_max_val = 1u << 14; + if (ramSize_GB >= k_max_val) + valMax = k_max_val; + else if (ramSize_GB > 1) + valMax = (UINT)ramSize_GB - 1; + else + valMax = 1; + } + + UInt32 limit = NExtract::Read_LimitGB(); + if (limit != 0 && limit != (UInt32)(Int32)-1) + CheckButton(IDX_SETTINGS_MEM_SET, true); + else + { + limit = 4; + EnableSpin(false); + } + SendItemMessage(IDC_SETTINGS_MEM_SPIN, UDM_SETRANGE, 0, MAKELPARAM(valMax, valMin)); // Sets the controls direction + // UDM_SETPOS doesn't set value larger than max value (valMax) of range: + SendItemMessage(IDC_SETTINGS_MEM_SPIN, UDM_SETPOS, 0, limit); + s.Empty(); + s.Add_UInt32(limit); + SetItemText(IDE_SETTINGS_MEM_SPIN_EDIT, s); + } + + _initMode = false; return CPropertyPage::OnInit(); } + +void CSettingsPage::EnableSpin(bool enable) +{ + EnableItem(IDC_SETTINGS_MEM_SPIN, enable); + EnableItem(IDE_SETTINGS_MEM_SPIN_EDIT, enable); +} + + /* void CSettingsPage::EnableSubItems() { @@ -237,6 +301,26 @@ LONG CSettingsPage::OnApply() } #endif + if (_memx_wasChanged) + { + UInt32 val = (UInt32)(Int32)-1; + if (IsButtonCheckedBool(IDX_SETTINGS_MEM_SET)) + { + UString s; + GetItemText(IDE_SETTINGS_MEM_SPIN_EDIT, s); + const wchar_t *end; + val = ConvertStringToUInt32(s.Ptr(), &end); + if (s.IsEmpty() || *end != 0 || val > (1u << 30)) + { + // L"Incorrect value" + ShowErrorMessage(*this, NError::MyFormatMessage(E_INVALIDARG)); + return PSNRET_INVALID; + } + } + NExtract::Save_LimitGB(val); + _memx_wasChanged = false; + } + /* if (_wasChanged_MemLimit) { @@ -304,24 +388,32 @@ void CSettingsPage::OnNotifyHelp() ShowHelpWindow(kSettingsTopic); } -/* bool CSettingsPage::OnCommand(unsigned code, unsigned itemID, LPARAM param) { - if (code == CBN_SELCHANGE) + if (!_initMode) { - switch (itemID) + if (code == EN_CHANGE && itemID == IDE_SETTINGS_MEM_SPIN_EDIT) + { + _memx_wasChanged = true; + Changed(); + } + /* + if (code == CBN_SELCHANGE) { - case IDC_SETTINGS_MEM: + switch (itemID) { - _wasChanged_MemLimit = true; - Changed(); - break; + case IDC_SETTINGS_MEM: + { + _wasChanged_MemLimit = true; + Changed(); + break; + } } } + */ } return CPropertyPage::OnCommand(code, itemID, param); } -*/ bool CSettingsPage::OnButtonClicked(unsigned buttonID, HWND buttonHWND) { @@ -345,6 +437,13 @@ bool CSettingsPage::OnButtonClicked(unsigned buttonID, HWND buttonHWND) _largePages_wasChanged = true; break; + case IDX_SETTINGS_MEM_SET: + { + _memx_wasChanged = true; + EnableSpin(IsButtonCheckedBool(IDX_SETTINGS_MEM_SET)); + break; + } + default: return CPropertyPage::OnButtonClicked(buttonID, buttonHWND); } diff --git a/CPP/7zip/UI/FileManager/SettingsPage.h b/CPP/7zip/UI/FileManager/SettingsPage.h index 91b9828..586b253 100644 --- a/CPP/7zip/UI/FileManager/SettingsPage.h +++ b/CPP/7zip/UI/FileManager/SettingsPage.h @@ -4,13 +4,15 @@ #define ZIP7_INC_SETTINGS_PAGE_H #include "../../../Windows/Control/PropertyPage.h" -#include "../../../Windows/Control/ComboBox.h" +// #include "../../../Windows/Control/ComboBox.h" #include "../../../Windows/Control/Edit.h" class CSettingsPage: public NWindows::NControl::CPropertyPage { bool _wasChanged; bool _largePages_wasChanged; + bool _memx_wasChanged; + bool _initMode; /* bool _wasChanged_MemLimit; NWindows::NControl::CComboBox _memCombo; @@ -22,11 +24,13 @@ class CSettingsPage: public NWindows::NControl::CPropertyPage */ // void EnableSubItems(); - // bool OnCommand(unsigned code, unsigned itemID, LPARAM param) Z7_override; + bool OnCommand(unsigned code, unsigned itemID, LPARAM param) Z7_override; virtual bool OnButtonClicked(unsigned buttonID, HWND buttonHWND) Z7_override; virtual bool OnInit() Z7_override; virtual void OnNotifyHelp() Z7_override; virtual LONG OnApply() Z7_override; + + void EnableSpin(bool enable); public: }; diff --git a/CPP/7zip/UI/FileManager/SettingsPage.rc b/CPP/7zip/UI/FileManager/SettingsPage.rc index baab484..59ea5ee 100644 --- a/CPP/7zip/UI/FileManager/SettingsPage.rc +++ b/CPP/7zip/UI/FileManager/SettingsPage.rc @@ -1,8 +1,8 @@ #include "SettingsPageRes.h" #include "../../GuiCommon.rc" -#define xc 240 -#define yc 250 +#define xc OPTIONS_PAGE_XC_SIZE +#define yc OPTIONS_PAGE_YC_SIZE IDD_SETTINGS MY_PAGE #include "SettingsPage2.rc" diff --git a/CPP/7zip/UI/FileManager/SettingsPage2.rc b/CPP/7zip/UI/FileManager/SettingsPage2.rc index cf90742..f67ee80 100644 --- a/CPP/7zip/UI/FileManager/SettingsPage2.rc +++ b/CPP/7zip/UI/FileManager/SettingsPage2.rc @@ -1,19 +1,37 @@ // #define g1xs 60 +// #include "MemDialogRes.h" + +#define save_y 144 +#define spin_y (save_y + 12) +#define spin_x_size 50 CAPTION "Settings" -BEGIN - CONTROL "Show "".."" item", IDX_SETTINGS_SHOW_DOTS, MY_CHECKBOX, m, 8, xc, 10 - CONTROL "Show real file &icons", IDX_SETTINGS_SHOW_REAL_FILE_ICONS, MY_CHECKBOX, m, 22, xc, 10 - CONTROL "&Full row select", IDX_SETTINGS_FULL_ROW, MY_CHECKBOX, m, 36, xc, 10 - CONTROL "Show &grid lines", IDX_SETTINGS_SHOW_GRID, MY_CHECKBOX, m, 50, xc, 10 - CONTROL "&Single-click to open an item", IDX_SETTINGS_SINGLE_CLICK, MY_CHECKBOX, m, 64, xc, 10 - CONTROL "&Alternative selection mode", IDX_SETTINGS_ALTERNATIVE_SELECTION, MY_CHECKBOX, m, 78, xc, 10 +BEGIN + MY_CONTROL_CHECKBOX ( "Show "".."" item", IDX_SETTINGS_SHOW_DOTS, m, 8, xc) + MY_CONTROL_CHECKBOX ( "Show real file &icons", IDX_SETTINGS_SHOW_REAL_FILE_ICONS, m, 22, xc) + MY_CONTROL_CHECKBOX ( "&Full row select", IDX_SETTINGS_FULL_ROW, m, 36, xc) + MY_CONTROL_CHECKBOX ( "Show &grid lines", IDX_SETTINGS_SHOW_GRID, m, 50, xc) + MY_CONTROL_CHECKBOX ( "&Single-click to open an item", IDX_SETTINGS_SINGLE_CLICK, m, 64, xc) + MY_CONTROL_CHECKBOX ( "&Alternative selection mode", IDX_SETTINGS_ALTERNATIVE_SELECTION, m, 78, xc) + + MY_CONTROL_CHECKBOX ( "Show system &menu", IDX_SETTINGS_SHOW_SYSTEM_MENU, m, 100, xc) - CONTROL "Show system &menu", IDX_SETTINGS_SHOW_SYSTEM_MENU, MY_CHECKBOX, m, 100, xc, 10 + MY_CONTROL_CHECKBOX ( "Use &large memory pages", IDX_SETTINGS_LARGE_PAGES, m, 122, xc) - CONTROL "Use &large memory pages", IDX_SETTINGS_LARGE_PAGES, MY_CHECKBOX, m, 122, xc, 10 + LTEXT "Maximum amount of RAM memory usage allowed to unpack archives:", + IDT_MEM_USAGE_EXTRACT, m, save_y, xc, 8 + MY_CONTROL_CHECKBOX_COLON(IDX_SETTINGS_MEM_SET, m + 10, spin_y + 1) + MY_CONTROL_EDIT_WITH_SPIN( + IDE_SETTINGS_MEM_SPIN_EDIT, + IDC_SETTINGS_MEM_SPIN, + "4", m + 30, spin_y, spin_x_size) + LTEXT "GB", IDT_SETTINGS_MEM_GB, m + 30 + spin_x_size + 8, spin_y + 2, xc - m - 30 - spin_x_size - 8, 8 // LTEXT "Memory usage for Compressing:", IDT_COMPRESS_MEMORY, m, 140, xc, 8 // COMBOBOX IDC_SETTINGS_MEM, m , 152, g1xs, yc - 152, MY_COMBO // LTEXT "/ RAM", IDT_SETTINGS_MEM_RAM, m + g1xs + m, 154, xc - g1xs - m, MY_TEXT_NOPREFIX END + +#undef save_y +#undef spin_y +#undef spin_x_size diff --git a/CPP/7zip/UI/FileManager/SettingsPageRes.h b/CPP/7zip/UI/FileManager/SettingsPageRes.h index e990bab..205daee 100644 --- a/CPP/7zip/UI/FileManager/SettingsPageRes.h +++ b/CPP/7zip/UI/FileManager/SettingsPageRes.h @@ -10,6 +10,12 @@ #define IDX_SETTINGS_ALTERNATIVE_SELECTION 2507 #define IDX_SETTINGS_LARGE_PAGES 2508 +#define IDT_MEM_USAGE_EXTRACT 7816 + +#define IDX_SETTINGS_MEM_SET 100 +#define IDE_SETTINGS_MEM_SPIN_EDIT 101 +#define IDC_SETTINGS_MEM_SPIN 102 +#define IDT_SETTINGS_MEM_GB 103 // #define IDT_SETTINGS_MEM 100 // #define IDC_SETTINGS_MEM 101 diff --git a/CPP/7zip/UI/FileManager/StdAfx.h b/CPP/7zip/UI/FileManager/StdAfx.h index 789cc6e..7790eae 100644 --- a/CPP/7zip/UI/FileManager/StdAfx.h +++ b/CPP/7zip/UI/FileManager/StdAfx.h @@ -7,23 +7,7 @@ #pragma warning(disable : 4464) // relative include path contains '..' #endif -#include "../../../../C/Compiler.h" - -Z7_DIAGNOSCTIC_IGNORE_BEGIN_RESERVED_MACRO_IDENTIFIER -#ifndef _WIN32_WINNT -// #define _WIN32_WINNT 0x0400 -#define _WIN32_WINNT 0x0500 -// #define _WIN32_WINNT 0x0600 -// #define _WIN32_WINNT 0x0A00 -#endif -#ifndef WINVER -#define WINVER _WIN32_WINNT -#endif -// #define _WIN32_IE 0x400 // for debug -Z7_DIAGNOSCTIC_IGNORE_END_RESERVED_MACRO_IDENTIFIER - #include "../../../Common/Common.h" -#include "../../../Common/MyWindows.h" #endif diff --git a/CPP/7zip/UI/FileManager/SysIconUtils.cpp b/CPP/7zip/UI/FileManager/SysIconUtils.cpp index 1c7cab0..c893ea9 100644 --- a/CPP/7zip/UI/FileManager/SysIconUtils.cpp +++ b/CPP/7zip/UI/FileManager/SysIconUtils.cpp @@ -49,6 +49,7 @@ int GetIconIndexForCSIDL(int csidl) } #ifndef _UNICODE +Z7_DIAGNOSTIC_IGNORE_CAST_FUNCTION typedef DWORD_PTR (WINAPI * Func_SHGetFileInfoW)(LPCWSTR pszPath, DWORD attrib, SHFILEINFOW *psfi, UINT cbFileInfo, UINT uFlags); static struct C_SHGetFileInfo_Init diff --git a/CPP/7zip/UI/FileManager/SystemPage.cpp b/CPP/7zip/UI/FileManager/SystemPage.cpp index 09d8a72..efa98a8 100644 --- a/CPP/7zip/UI/FileManager/SystemPage.cpp +++ b/CPP/7zip/UI/FileManager/SystemPage.cpp @@ -171,7 +171,7 @@ bool CSystemPage::OnInit() _listView.SetImageList(_imageList, LVSIL_SMALL); - _listView.InsertColumn(0, LangString(IDS_PROP_FILE_TYPE), 72); + _listView.InsertColumn(0, LangString(IDS_PROP_FILE_TYPE), 80); UString s; @@ -206,7 +206,7 @@ bool CSystemPage::OnInit() LV_COLUMNW ci; ci.mask = LVCF_TEXT | LVCF_FMT | LVCF_WIDTH | LVCF_SUBITEM; - ci.cx = 128; + ci.cx = 152; ci.fmt = LVCFMT_CENTER; ci.pszText = s.Ptr_non_const(); ci.iSubItem = 1; diff --git a/CPP/7zip/UI/FileManager/SystemPage.rc b/CPP/7zip/UI/FileManager/SystemPage.rc index 3bb143a..108e524 100644 --- a/CPP/7zip/UI/FileManager/SystemPage.rc +++ b/CPP/7zip/UI/FileManager/SystemPage.rc @@ -1,15 +1,15 @@ #include "SystemPageRes.h" #include "../../GuiCommon.rc" -#define xc 240 -#define yc 252 +#define xc OPTIONS_PAGE_XC_SIZE +#define yc OPTIONS_PAGE_YC_SIZE -IDD_SYSTEM DIALOG 0, 0, xs, ys MY_PAGE_STYLE MY_FONT +IDD_SYSTEM DIALOG MY_PAGE_POSTFIX CAPTION "System" BEGIN LTEXT "Associate 7-Zip with:", IDT_SYSTEM_ASSOCIATE, m, m, xc, 8 - PUSHBUTTON "+", IDB_SYSTEM_CURRENT, 80, m + 12, 40, bys - PUSHBUTTON "+", IDB_SYSTEM_ALL, 166, m + 12, 40, bys + PUSHBUTTON "+", IDB_SYSTEM_CURRENT, 92, m + 12, 40, bys + PUSHBUTTON "+", IDB_SYSTEM_ALL, 194, m + 12, 40, bys CONTROL "List1", IDL_SYSTEM_ASSOCIATE, "SysListView32", LVS_REPORT | LVS_SHOWSELALWAYS | LVS_SHAREIMAGELISTS | LVS_NOSORTHEADER | WS_BORDER | WS_TABSTOP, m, m + 32, xc, (yc - 32) @@ -25,7 +25,7 @@ END #define xc (SMALL_PAGE_SIZE_X + 8) #define yc (128 + 8) -IDD_SYSTEM_2 DIALOG 0, 0, xs, ys MY_PAGE_STYLE MY_FONT +IDD_SYSTEM_2 DIALOG MY_PAGE_POSTFIX CAPTION "System" BEGIN LTEXT "Associate 7-Zip with:", IDT_SYSTEM_ASSOCIATE, m, m, xc, 8 diff --git a/CPP/7zip/UI/FileManager/TextPairs.cpp b/CPP/7zip/UI/FileManager/TextPairs.cpp index 1ac7098..6bf84cf 100644 --- a/CPP/7zip/UI/FileManager/TextPairs.cpp +++ b/CPP/7zip/UI/FileManager/TextPairs.cpp @@ -181,13 +181,13 @@ void CPairsStorage::SaveToString(UString &text) const const CTextPair &pair = Pairs[i]; bool multiWord = (pair.ID.Find(L' ') >= 0); if (multiWord) - text += '\"'; + text.Add_Char('\"'); text += pair.ID; if (multiWord) - text += '\"'; + text.Add_Char('\"'); text.Add_Space(); text += pair.Value; - text += '\x0D'; + text.Add_Char('\x0D'); text.Add_LF(); } } diff --git a/CPP/7zip/UI/FileManager/VerCtrl.cpp b/CPP/7zip/UI/FileManager/VerCtrl.cpp index 0c894b8..f1353b8 100644 --- a/CPP/7zip/UI/FileManager/VerCtrl.cpp +++ b/CPP/7zip/UI/FileManager/VerCtrl.cpp @@ -93,7 +93,7 @@ static bool ParseNumberString(const FString &s, UInt32 &number) static void WriteFile(const FString &path, bool createAlways, const CFileDataInfo &fdi, const CPanel &panel) { NIO::COutFile outFile; - if (!outFile.Create(path, createAlways)) // (createAlways = false) means CREATE_NEW + if (!outFile.Create_ALWAYS_or_NEW(path, createAlways)) // (createAlways = false) means CREATE_NEW { panel.MessageBox_LastError(); return; @@ -339,7 +339,7 @@ void CApp::VerCtrl(unsigned id) // NDir::SetFileAttrib(path, 0); { NIO::COutFile outFile; - if (!outFile.Open(path, OPEN_EXISTING)) + if (!outFile.Open_EXISTING(path)) { panel.MessageBox_LastError(); return; diff --git a/CPP/7zip/UI/FileManager/makefile b/CPP/7zip/UI/FileManager/makefile index df08c98..0ca5caa 100644 --- a/CPP/7zip/UI/FileManager/makefile +++ b/CPP/7zip/UI/FileManager/makefile @@ -2,6 +2,8 @@ PROG = 7zFM.exe CFLAGS = $(CFLAGS) \ -DZ7_EXTERNAL_CODECS \ +# -DZ7_NO_LARGE_PAGES + !include "FM.mak" COMMON_OBJS = \ diff --git a/CPP/7zip/UI/FileManager/resource.h b/CPP/7zip/UI/FileManager/resource.h index 9d605c6..4e22230 100644 --- a/CPP/7zip/UI/FileManager/resource.h +++ b/CPP/7zip/UI/FileManager/resource.h @@ -25,6 +25,8 @@ #define IDM_CRC64 103 #define IDM_SHA1 104 #define IDM_SHA256 105 +#define IDM_XXH64 106 +#define IDM_BLAKE2SP 107 #define IDM_FILE 500 @@ -100,6 +102,7 @@ #define IDM_VIEW_TIME_POPUP 760 #define IDM_VIEW_TIME 761 +#define IDM_VIEW_TIME_UTC 799 #define IDM_ADD_TO_FAVORITES 800 #define IDS_BOOKMARK 801 @@ -107,6 +110,7 @@ #define IDM_OPTIONS 900 #define IDM_BENCHMARK 901 #define IDM_BENCHMARK2 902 +#define IDM_TEMP_DIR 910 #define IDM_HELP_CONTENTS 960 #define IDM_ABOUT 961 diff --git a/CPP/7zip/UI/FileManager/resource.rc b/CPP/7zip/UI/FileManager/resource.rc index 002265a..feeeaf5 100644 --- a/CPP/7zip/UI/FileManager/resource.rc +++ b/CPP/7zip/UI/FileManager/resource.rc @@ -57,8 +57,10 @@ BEGIN BEGIN MENUITEM "CRC-32", IDM_CRC32 MENUITEM "CRC-64", IDM_CRC64 + MENUITEM "XXH64", IDM_XXH64 MENUITEM "SHA-1", IDM_SHA1 MENUITEM "SHA-256", IDM_SHA256 + MENUITEM "BLAKE2sp", IDM_BLAKE2SP MENUITEM "*", IDM_HASH_ALL END MENUITEM "Di&ff", IDM_DIFF @@ -141,6 +143,8 @@ BEGIN #ifdef UNDER_CE MENUITEM "Benchmark 2", IDM_BENCHMARK2 #endif + MY_MENUITEM_SEPARATOR + MENUITEM "Delete Temporary Files...", IDM_TEMP_DIR #ifndef UNDER_CE END POPUP "&Help" MY_MENUITEM_ID(IDM_HELP) @@ -263,6 +267,7 @@ END #include "AboutDialog.rc" #include "BrowseDialog.rc" +#include "BrowseDialog2.rc" #include "ComboDialog.rc" #include "CopyDialog.rc" #include "EditDialog.rc" @@ -271,6 +276,7 @@ END #include "LangPage.rc" #include "LinkDialog.rc" #include "ListViewDialog.rc" +#include "MemDialog.rc" #include "MenuPage.rc" #include "MessagesDialog.rc" #include "OverwriteDialog.rc" diff --git a/CPP/7zip/UI/FileManager/resourceGui.h b/CPP/7zip/UI/FileManager/resourceGui.h index 7c1b40e..848b36f 100644 --- a/CPP/7zip/UI/FileManager/resourceGui.h +++ b/CPP/7zip/UI/FileManager/resourceGui.h @@ -13,3 +13,11 @@ #define IDS_CHECKSUM_CRC_STREAMS_NAMES 7504 #define IDS_INCORRECT_VOLUME_SIZE 7307 + +#define IDS_MEM_REQUIRES_BIG_MEM 7811 +#define IDS_MEM_REQUIRED_MEM_SIZE 7812 +#define IDS_MEM_CURRENT_MEM_LIMIT 7813 +#define IDS_MEM_USAGE_LIMIT_SET_BY_7ZIP 7814 +#define IDS_MEM_RAM_SIZE 7815 + +#define IDS_MSG_ARC_UNPACKING_WAS_SKIPPED 7822 diff --git a/CPP/7zip/UI/FileManager/resourceGui.rc b/CPP/7zip/UI/FileManager/resourceGui.rc index f748e0b..143e9f6 100644 --- a/CPP/7zip/UI/FileManager/resourceGui.rc +++ b/CPP/7zip/UI/FileManager/resourceGui.rc @@ -16,4 +16,13 @@ BEGIN IDS_OPENNING "Opening..." IDS_SCANNING "Scanning..." + IDS_MEM_REQUIRES_BIG_MEM "The operation requires big amount of memory (RAM)." + IDS_MEM_REQUIRED_MEM_SIZE "required memory usage size" + IDS_MEM_CURRENT_MEM_LIMIT "allowed memory usage limit" + IDS_MEM_USAGE_LIMIT_SET_BY_7ZIP "memory usage limit set by 7-Zip" + IDS_MEM_RAM_SIZE "RAM size" + + IDS_MSG_ARC_UNPACKING_WAS_SKIPPED "Archive extraction was skipped." +// IDS_MSG_ARC_FILES_UNPACKING_WAS_SKIPPED "Files extraction was skipped." + END diff --git a/CPP/7zip/UI/GUI/BenchmarkDialog.cpp b/CPP/7zip/UI/GUI/BenchmarkDialog.cpp index 539d689..7f2edfa 100644 --- a/CPP/7zip/UI/GUI/BenchmarkDialog.cpp +++ b/CPP/7zip/UI/GUI/BenchmarkDialog.cpp @@ -23,6 +23,7 @@ #include "../FileManager/DialogSize.h" #include "../FileManager/HelpUtils.h" #include "../FileManager/LangUtils.h" +#include "../FileManager/resourceGui.h" #include "../../MyVersion.h" @@ -884,9 +885,15 @@ void CBenchmarkDialog::StartBenchmark() false); // totalBench if (!IsMemoryUsageOK(memUsage)) { - UString s2 = LangString(IDT_BENCH_MEMORY); + UString s2; + LangString_OnlyFromLangFile(IDS_MEM_REQUIRED_MEM_SIZE, s2); if (s2.IsEmpty()) - GetItemText(IDT_BENCH_MEMORY, s2); + { + s2 = LangString(IDT_BENCH_MEMORY); + if (s2.IsEmpty()) + GetItemText(IDT_BENCH_MEMORY, s2); + s2.RemoveChar(L':'); + } UString s; SetErrorMessage_MemUsage(s, memUsage, RamSize, RamSize_Limit, s2); MessageBoxError_Status(s); @@ -1081,7 +1088,7 @@ static void Add_Dot3String(UString &s, UInt64 val) static void AddRatingString(UString &s, const CTotalBenchRes &info) { // AddUsageString(s, info); - // s += " "; + // s.Add_Space(); // s.Add_UInt32(GetRating(info)); Add_Dot3String(s, GetRating(info)); } @@ -1093,7 +1100,7 @@ static void AddRatingsLine(UString &s, const CTotalBenchRes &enc, const CTotalBe #endif ) { - // AddUsageString(s, enc); s += " "; + // AddUsageString(s, enc); s.Add_Space(); AddRatingString(s, enc); s += " "; @@ -1105,11 +1112,11 @@ static void AddRatingsLine(UString &s, const CTotalBenchRes &enc, const CTotalBe s += " "; AddRatingString(s, tot_BenchRes); - s += " "; AddUsageString(s, tot_BenchRes); + s.Add_Space(); AddUsageString(s, tot_BenchRes); #ifdef PRINT_ITER_TIME - s += " "; + s.Add_Space(); { Add_Dot3String(s, ticks; s += " s"; @@ -1342,7 +1349,7 @@ void CBenchmarkDialog::UpdateGui() /* s += "g:"; s.Add_UInt32((UInt32)pair.EncInfo.GlobalTime); s += " u:"; s.Add_UInt32((UInt32)pair.EncInfo.UserTime); - s += " "; + s.Add_Space(); */ AddRatingsLine(s, pair.Enc, pair.Dec #ifdef PRINT_ITER_TIME @@ -1547,11 +1554,11 @@ HRESULT CFreqCallback::AddCpuFreq(unsigned numThreads, UInt64 freq, UInt64 usage s += "T Frequency (MHz):"; s.Add_LF(); } - s += " "; + s.Add_Space(); if (numThreads != 1) { s.Add_UInt64(GetUsagePercents(usage)); - s += '%'; + s.Add_Char('%'); s.Add_Space(); } s.Add_UInt64(GetMips(freq)); @@ -1668,7 +1675,7 @@ HRESULT CThreadBenchmark::Process() CProperty prop; prop.Name = 'd'; prop.Name.Add_UInt32((UInt32)(dictionarySize >> 10)); - prop.Name += 'k'; + prop.Name.Add_Char('k'); props.Add(prop); } } @@ -1879,7 +1886,7 @@ HRESULT Benchmark( { // bd.Bench2Text.Empty(); bd.Bench2Text = "7-Zip " MY_VERSION_CPU; - bd.Bench2Text += (char)0xD; + // bd.Bench2Text.Add_Char((char)0xD); bd.Bench2Text.Add_LF(); } diff --git a/CPP/7zip/UI/GUI/CompressDialog.cpp b/CPP/7zip/UI/GUI/CompressDialog.cpp index c2aee6c..fd53062 100644 --- a/CPP/7zip/UI/GUI/CompressDialog.cpp +++ b/CPP/7zip/UI/GUI/CompressDialog.cpp @@ -18,6 +18,7 @@ #include "../FileManager/HelpUtils.h" #include "../FileManager/PropertyName.h" #include "../FileManager/SplitUtils.h" +#include "../FileManager/resourceGui.h" #include "../Explorer/MyMessages.h" @@ -1015,41 +1016,34 @@ static bool IsAsciiString(const UString &s) static void AddSize_MB(UString &s, UInt64 size) { + s.Add_LF(); const UInt64 v2 = size + ((UInt32)1 << 20) - 1; - if (size <= v2) - size = v2; + if (size < v2) + size = v2; s.Add_UInt64(size >> 20); - s += " MB"; + s += " MB : "; } +static void AddSize_MB_id(UString &s, UInt64 size, UInt32 id) +{ + AddSize_MB(s, size); + AddLangString(s, id); +} void SetErrorMessage_MemUsage(UString &s, UInt64 reqSize, UInt64 ramSize, UInt64 ramLimit, const UString &usageString); void SetErrorMessage_MemUsage(UString &s, UInt64 reqSize, UInt64 ramSize, UInt64 ramLimit, const UString &usageString) { - s += "The operation was blocked by 7-Zip"; - s.Add_LF(); - s += "The operation can require big amount of RAM (memory):"; + AddLangString(s, IDS_MEM_OPERATION_BLOCKED); s.Add_LF(); + AddLangString(s, IDS_MEM_REQUIRES_BIG_MEM); s.Add_LF(); AddSize_MB(s, reqSize); - - if (!usageString.IsEmpty()) - { - s += " : "; - s += usageString; - } - - s.Add_LF(); - AddSize_MB(s, ramSize); - s += " : RAM"; - + s += usageString; + AddSize_MB_id(s, ramSize, IDS_MEM_RAM_SIZE); // if (ramLimit != 0) { - s.Add_LF(); - AddSize_MB(s, ramLimit); - s += " : 7-Zip limit"; + AddSize_MB_id(s, ramLimit, IDS_MEM_USAGE_LIMIT_SET_BY_7ZIP); } - s.Add_LF(); s.Add_LF(); AddLangString(s, IDS_MEM_ERROR); @@ -1095,10 +1089,16 @@ void CCompressDialog::OnOK() const UInt64 limit = Get_MemUse_Bytes(); if (memUsage > limit) { - UString s; - UString s2 = LangString(IDT_COMPRESS_MEMORY); + UString s2; + LangString_OnlyFromLangFile(IDS_MEM_REQUIRED_MEM_SIZE, s2); if (s2.IsEmpty()) - GetItemText(IDT_COMPRESS_MEMORY, s2); + { + s2 = LangString(IDT_COMPRESS_MEMORY); + if (s2.IsEmpty()) + GetItemText(IDT_COMPRESS_MEMORY, s2); + s2.RemoveChar(L':'); + } + UString s; SetErrorMessage_MemUsage(s, memUsage, _ramSize, limit, s2); MessageBoxError(s); return; @@ -1823,8 +1823,8 @@ static int Combo_AddDict2(NWindows::NControl::CComboBox &cb, size_t sizeReal, si s.Add_UInt64(sizeShow >> moveBits); s.Add_Space(); if (c != 0) - s += c; - s += 'B'; + s.Add_Char(c); + s.Add_Char('B'); if (sizeReal == k_Auto_Dict) Modify_Auto(s); const int index = (int)ComboBox_AddStringAscii(cb, s); @@ -2394,8 +2394,8 @@ static void Add_Size(AString &s, UInt64 val) s.Add_UInt64(val >> moveBits); s.Add_Space(); if (moveBits != 0) - s += c; - s += 'B'; + s.Add_Char(c); + s.Add_Char('B'); } @@ -2714,8 +2714,8 @@ static void AddMemSize(UString &res, UInt64 size) res.Add_UInt64(size >> moveBits); res.Add_Space(); if (moveBits != 0) - res += c; - res += 'B'; + res.Add_Char(c); + res.Add_Char('B'); } @@ -2727,7 +2727,7 @@ int CCompressDialog::AddMemComboItem(UInt64 val, bool isPercent, bool isDefault) { UString s; s.Add_UInt64(val); - s += '%'; + s.Add_Char('%'); if (isDefault) sUser = k_Auto_Prefix; else @@ -3510,7 +3510,7 @@ void COptionsDialog::SetPrec() // defaultPrec = kTimePrec_Unix; // flags = (UInt32)1 << kTimePrec_Unix; - s += ":"; + s.Add_Colon(); if (methodID >= 0 && (unsigned)methodID < Z7_ARRAY_SIZE(kMethodsNames)) s += kMethodsNames[methodID]; if (methodID == kPosix) diff --git a/CPP/7zip/UI/GUI/CompressDialog.rc b/CPP/7zip/UI/GUI/CompressDialog.rc index f04329e..9c3ed88 100644 --- a/CPP/7zip/UI/GUI/CompressDialog.rc +++ b/CPP/7zip/UI/GUI/CompressDialog.rc @@ -225,6 +225,8 @@ BEGIN IDS_COMPRESS_SEC "sec" IDS_COMPRESS_NS "ns" + + IDS_MEM_OPERATION_BLOCKED "The operation was blocked by 7-Zip." END diff --git a/CPP/7zip/UI/GUI/CompressDialogRes.h b/CPP/7zip/UI/GUI/CompressDialogRes.h index aa25b89..d04d4b9 100644 --- a/CPP/7zip/UI/GUI/CompressDialogRes.h +++ b/CPP/7zip/UI/GUI/CompressDialogRes.h @@ -121,3 +121,5 @@ // #define IDX_COMPRESS_NT_HARD_LINKS_SET 211 // #define IDX_COMPRESS_NT_ALT_STREAMS_SET 212 // #define IDX_COMPRESS_NT_SECUR_SET 213 + +#define IDS_MEM_OPERATION_BLOCKED 7810 diff --git a/CPP/7zip/UI/GUI/CompressOptionsDialog.rc b/CPP/7zip/UI/GUI/CompressOptionsDialog.rc index 0578227..682775e 100644 --- a/CPP/7zip/UI/GUI/CompressOptionsDialog.rc +++ b/CPP/7zip/UI/GUI/CompressOptionsDialog.rc @@ -17,57 +17,60 @@ #define ntSizeY 72 #define timePosY (ntPosY + ntSizeY + 20) -#define ceSize 18 -#define ceString ":" - IDD_COMPRESS_OPTIONS DIALOG 0, 0, xs, ys MY_MODAL_DIALOG_STYLE MY_FONT CAPTION "Options" BEGIN GROUPBOX "NTFS", IDG_COMPRESS_NTFS, g5x, ntPosY, g5xs, ntSizeY - CONTROL "Store symbolic links", IDX_COMPRESS_NT_SYM_LINKS, MY_CHECKBOX, - ntPosX, ntPosY + 12, ntSizeX, 10 - CONTROL "Store hard links", IDX_COMPRESS_NT_HARD_LINKS, MY_CHECKBOX, - ntPosX, ntPosY + 26, ntSizeX, 10 - CONTROL "Store alternate data streams", IDX_COMPRESS_NT_ALT_STREAMS, MY_CHECKBOX, - ntPosX, ntPosY + 40, ntSizeX, 10 - CONTROL "Store file security", IDX_COMPRESS_NT_SECUR, MY_CHECKBOX, - ntPosX, ntPosY + 54, ntSizeX, 10 + MY_CONTROL_CHECKBOX ("Store symbolic links", IDX_COMPRESS_NT_SYM_LINKS, + ntPosX, ntPosY + 12, ntSizeX) + MY_CONTROL_CHECKBOX ("Store hard links", IDX_COMPRESS_NT_HARD_LINKS, + ntPosX, ntPosY + 26, ntSizeX) + MY_CONTROL_CHECKBOX ("Store alternate data streams", IDX_COMPRESS_NT_ALT_STREAMS, + ntPosX, ntPosY + 40, ntSizeX) + MY_CONTROL_CHECKBOX ("Store file security", IDX_COMPRESS_NT_SECUR, + ntPosX, ntPosY + 54, ntSizeX) LTEXT "", IDT_COMPRESS_TIME_INFO, g5x, timePosY - 14, g5xs, 8 GROUPBOX "Time", IDG_COMPRESS_TIME, g5x, timePosY, g5xs, 112 -// CONTROL "Default", IDX_COMPRESS_TIME_DEFAULT, MY_CHECKBOX, -// ntPosX, timePosY + 10, ntSizeX, 16 +// MY_CONTROL_CHECKBOX ("Default", IDX_COMPRESS_TIME_DEFAULT, +// ntPosX, timePosY + 10, ntSizeX) - CONTROL ceString, IDX_COMPRESS_PREC_SET, MY_CHECKBOX, ntPosX, timePosY + 14, ceSize, 10 + MY_CONTROL_CHECKBOX_COLON (IDX_COMPRESS_PREC_SET, ntPosX, timePosY + 14) LTEXT "Timestamp precision:", IDT_COMPRESS_TIME_PREC, - ntPosX + ceSize, timePosY + 14, ntSizeX - precSizeX - ceSize, 8 + ntPosX + cboxColonSize, timePosY + 14, ntSizeX - precSizeX - cboxColonSize, 8 COMBOBOX IDC_COMPRESS_TIME_PREC, ntPosX + ntSizeX - precSizeX, timePosY + 12, precSizeX, 70, MY_COMBO // PUSHBUTTON "Default", IDB_COMPRESS_TIME_DEFAULT, ntPosX + ntSizeX - bxs, timePosY + 22, bxs, bys, WS_GROUP - CONTROL ceString, IDX_COMPRESS_MTIME_SET, MY_CHECKBOX, ntPosX, timePosY + 28, ceSize, 10 - CONTROL "Store modification time", IDX_COMPRESS_MTIME, MY_CHECKBOX, - ntPosX + ceSize, timePosY + 28, ntSizeX - ceSize, 10 - - CONTROL ceString, IDX_COMPRESS_CTIME_SET, MY_CHECKBOX, ntPosX, timePosY + 42, ceSize, 10 - CONTROL "Store creation time", IDX_COMPRESS_CTIME, MY_CHECKBOX, - ntPosX + ceSize, timePosY + 42, ntSizeX - ceSize, 10 - - CONTROL ceString, IDX_COMPRESS_ATIME_SET, MY_CHECKBOX, ntPosX, timePosY + 56, ceSize, 10 - CONTROL "Store last access time", IDX_COMPRESS_ATIME, MY_CHECKBOX, - ntPosX + ceSize, timePosY + 56, ntSizeX - ceSize, 10 - - CONTROL ceString, IDX_COMPRESS_ZTIME_SET, MY_CHECKBOX | BS_MULTILINE, ntPosX, timePosY + 72, ceSize, 16 - CONTROL "Set archive time to latest file time", IDX_COMPRESS_ZTIME, MY_CHECKBOX | BS_MULTILINE, - ntPosX + ceSize, timePosY + 72, ntSizeX - ceSize, 16 - - CONTROL "Do not change source files last access time", IDX_COMPRESS_PRESERVE_ATIME, MY_CHECKBOX | BS_MULTILINE, - ntPosX, timePosY + 92, ntSizeX, 16 + MY_CONTROL_CHECKBOX_COLON (IDX_COMPRESS_MTIME_SET, ntPosX, timePosY + 28) + MY_CONTROL_CHECKBOX ("Store modification time", IDX_COMPRESS_MTIME, + ntPosX + cboxColonSize, timePosY + 28, ntSizeX - cboxColonSize) + + MY_CONTROL_CHECKBOX_COLON (IDX_COMPRESS_CTIME_SET, ntPosX, timePosY + 42) + MY_CONTROL_CHECKBOX ("Store creation time", IDX_COMPRESS_CTIME, + ntPosX + cboxColonSize, timePosY + 42, ntSizeX - cboxColonSize) + + MY_CONTROL_CHECKBOX_COLON (IDX_COMPRESS_ATIME_SET, ntPosX, timePosY + 56) + MY_CONTROL_CHECKBOX ("Store last access time", IDX_COMPRESS_ATIME, + ntPosX + cboxColonSize, timePosY + 56, ntSizeX - cboxColonSize) + + MY_CONTROL_CHECKBOX_2LINES(colonString, + IDX_COMPRESS_ZTIME_SET, + ntPosX, timePosY + 72, cboxColonSize) + MY_CONTROL_CHECKBOX_2LINES( + "Set archive time to latest file time", + IDX_COMPRESS_ZTIME, + ntPosX + cboxColonSize, timePosY + 72, ntSizeX - cboxColonSize) + + MY_CONTROL_CHECKBOX_2LINES( + "Do not change source files last access time", + IDX_COMPRESS_PRESERVE_ATIME, + ntPosX, timePosY + 92, ntSizeX) DEFPUSHBUTTON "OK", IDOK, bx3, by, bxs, bys, WS_GROUP diff --git a/CPP/7zip/UI/GUI/ExtractGUI.cpp b/CPP/7zip/UI/GUI/ExtractGUI.cpp index fdf3cc7..bafff2b 100644 --- a/CPP/7zip/UI/GUI/ExtractGUI.cpp +++ b/CPP/7zip/UI/GUI/ExtractGUI.cpp @@ -42,7 +42,7 @@ static void AddValuePair(UString &s, UINT resourceID, UInt64 value, bool addColo { AddLangString(s, resourceID); if (addColon) - s += ':'; + s.Add_Colon(); s.Add_Space(); s.Add_UInt64(value); s.Add_LF(); diff --git a/CPP/7zip/UI/GUI/GUI.cpp b/CPP/7zip/UI/GUI/GUI.cpp index 37e637b..6bb6693 100644 --- a/CPP/7zip/UI/GUI/GUI.cpp +++ b/CPP/7zip/UI/GUI/GUI.cpp @@ -46,9 +46,19 @@ const CExternalCodecs *g_ExternalCodecs_Ptr; extern HINSTANCE g_hInstance; HINSTANCE g_hInstance; +extern +bool g_DisableUserQuestions; +bool g_DisableUserQuestions; #ifndef UNDER_CE +#if !defined(Z7_WIN32_WINNT_MIN) || Z7_WIN32_WINNT_MIN < 0x0500 // win2000 +#define Z7_USE_DYN_ComCtl32Version +#endif + +#ifdef Z7_USE_DYN_ComCtl32Version +Z7_DIAGNOSTIC_IGNORE_CAST_FUNCTION + extern DWORD g_ComCtl32Version; DWORD g_ComCtl32Version; @@ -77,6 +87,7 @@ static DWORD GetDllVersion(LPCTSTR dllName) return dwVersion; } +#endif #endif extern @@ -87,7 +98,8 @@ DECLARE_AND_SET_CLIENT_VERSION_VAR static void ErrorMessage(LPCWSTR message) { - MessageBoxW(NULL, message, L"7-Zip", MB_ICONERROR | MB_OK); + if (!g_DisableUserQuestions) + MessageBoxW(NULL, message, L"7-Zip", MB_ICONERROR | MB_OK); } static void ErrorMessage(const char *s) @@ -141,6 +153,7 @@ static int Main2() CArcCmdLineParser parser; parser.Parse1(commandStrings, options); + g_DisableUserQuestions = options.YesToAll; parser.Parse2(options); CREATE_CODECS_OBJECT @@ -157,7 +170,8 @@ static int Main2() codecs->GetCodecsErrorMessage(s); if (!s.IsEmpty()) { - MessageBoxW(NULL, s, L"7-Zip", MB_ICONERROR); + if (!g_DisableUserQuestions) + MessageBoxW(NULL, s, L"7-Zip", MB_ICONERROR); } } @@ -247,7 +261,9 @@ static int Main2() eo.StdInMode = options.StdInMode; eo.StdOutMode = options.StdOutMode; eo.YesToAll = options.YesToAll; + ecs->YesToAll = options.YesToAll; eo.TestMode = options.Command.IsTestCommand(); + ecs->TestMode = eo.TestMode; #ifndef Z7_SFX eo.Properties = options.Properties; @@ -405,10 +421,10 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE /* hPrevInstance */, InitCommonControls(); - #ifndef UNDER_CE +#ifdef Z7_USE_DYN_ComCtl32Version g_ComCtl32Version = ::GetDllVersion(TEXT("comctl32.dll")); g_LVN_ITEMACTIVATE_Support = (g_ComCtl32Version >= MAKELONG(71, 4)); - #endif +#endif // OleInitialize is required for ProgressBar in TaskBar. #ifndef UNDER_CE diff --git a/CPP/7zip/UI/GUI/GUI.dsp b/CPP/7zip/UI/GUI/GUI.dsp index 3a6b60f..1742f26 100644 --- a/CPP/7zip/UI/GUI/GUI.dsp +++ b/CPP/7zip/UI/GUI/GUI.dsp @@ -449,6 +449,14 @@ SOURCE=..\FileManager\ListViewDialog.h # End Source File # Begin Source File +SOURCE=..\FileManager\MemDialog.cpp +# End Source File +# Begin Source File + +SOURCE=..\FileManager\MemDialog.h +# End Source File +# Begin Source File + SOURCE=..\FileManager\OverwriteDialog.cpp # End Source File # Begin Source File @@ -781,6 +789,10 @@ SOURCE=..\..\..\..\C\7zTypes.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\7zVersion.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\7zWindows.h # End Source File # Begin Source File @@ -794,6 +806,10 @@ SOURCE=..\..\..\..\C\Alloc.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\Compiler.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\CpuArch.c !IF "$(CFG)" == "GUI - Win32 Release" @@ -831,6 +847,10 @@ SOURCE=..\..\..\..\C\DllSecur.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\Precomp.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Sort.c !IF "$(CFG)" == "GUI - Win32 Release" @@ -884,10 +904,6 @@ SOURCE=..\..\..\Common\Common.h # End Source File # Begin Source File -SOURCE=..\..\..\..\C\Compiler.h -# End Source File -# Begin Source File - SOURCE=..\..\..\Common\CRC.cpp # End Source File # Begin Source File diff --git a/CPP/7zip/UI/GUI/UpdateGUI.cpp b/CPP/7zip/UI/GUI/UpdateGUI.cpp index aaf7ebd..a600a8b 100644 --- a/CPP/7zip/UI/GUI/UpdateGUI.cpp +++ b/CPP/7zip/UI/GUI/UpdateGUI.cpp @@ -197,7 +197,7 @@ static void AddProp_Size(CObjectVector &properties, const char *name, { UString s; s.Add_UInt64(size); - s += 'b'; + s.Add_Char('b'); AddProp_UString(properties, name, s); } @@ -265,7 +265,7 @@ static void SetOutProperties( UString s; // s += 'p'; // for debug: alternate percent method s.Add_UInt64(memUse.Val); - s += '%'; + s.Add_Char('%'); AddProp_UString(properties, kMemUse, s); } else diff --git a/CPP/7zip/UI/GUI/makefile b/CPP/7zip/UI/GUI/makefile index 9e2a9b4..22ae095 100644 --- a/CPP/7zip/UI/GUI/makefile +++ b/CPP/7zip/UI/GUI/makefile @@ -7,7 +7,8 @@ CFLAGS = $(CFLAGS) \ LIBS = $(LIBS) ceshell.lib Commctrl.lib !ELSE LIBS = $(LIBS) comctl32.lib htmlhelp.lib comdlg32.lib gdi32.lib -CFLAGS = $(CFLAGS) -DZ7_LONG_PATH -DZ7_LARGE_PAGES -DZ7_DEVICE_FILE +CFLAGS = $(CFLAGS) -DZ7_DEVICE_FILE +# -DZ7_LONG_PATH -DZ7_LARGE_PAGES !ENDIF GUI_OBJS = \ @@ -114,6 +115,7 @@ FM_OBJS = \ $O\HelpUtils.obj \ $O\LangUtils.obj \ $O\ListViewDialog.obj \ + $O\MemDialog.obj \ $O\OpenCallback.obj \ $O\ProgramLocation.obj \ $O\PropertyName.obj \ diff --git a/CPP/7zip/UI/GUI/resource.rc b/CPP/7zip/UI/GUI/resource.rc index 04af815..08ef00a 100644 --- a/CPP/7zip/UI/GUI/resource.rc +++ b/CPP/7zip/UI/GUI/resource.rc @@ -15,6 +15,7 @@ IDI_ICON ICON "FM.ico" #endif #include "../FileManager/PropertyName.rc" +#include "../FileManager/MemDialog.rc" #include "../FileManager/OverwriteDialog.rc" #include "../FileManager/PasswordDialog.rc" #include "../FileManager/ProgressDialog2.rc" diff --git a/CPP/7zip/cmpl_gcc_arm.mak b/CPP/7zip/cmpl_gcc_arm.mak new file mode 100644 index 0000000..6671d58 --- /dev/null +++ b/CPP/7zip/cmpl_gcc_arm.mak @@ -0,0 +1,3 @@ +include ../../var_gcc_arm.mak +include ../../warn_gcc.mak +include makefile.gcc diff --git a/CPP/7zip/var_clang_arm64.mak b/CPP/7zip/var_clang_arm64.mak index 4b35409..ec81d26 100644 --- a/CPP/7zip/var_clang_arm64.mak +++ b/CPP/7zip/var_clang_arm64.mak @@ -4,8 +4,16 @@ IS_X64= IS_X86= IS_ARM64=1 CROSS_COMPILE= + +MY_ARCH=-march=armv8-a +MY_ARCH=-march=armv8-a+crypto+crc +MY_ARCH=-march=armv8.3-a+crypto+crc MY_ARCH= + +USE_ASM= USE_ASM=1 + +ASM_FLAGS=-Wno-unused-macros CC=$(CROSS_COMPILE)clang CXX=$(CROSS_COMPILE)clang++ USE_CLANG=1 diff --git a/CPP/7zip/var_clang_x64.mak b/CPP/7zip/var_clang_x64.mak index fefed51..34e1b49 100644 --- a/CPP/7zip/var_clang_x64.mak +++ b/CPP/7zip/var_clang_x64.mak @@ -9,4 +9,3 @@ USE_ASM=1 CC=$(CROSS_COMPILE)clang CXX=$(CROSS_COMPILE)clang++ USE_CLANG=1 - diff --git a/CPP/7zip/var_clang_x86.mak b/CPP/7zip/var_clang_x86.mak index 5f3c2d9..bd2317c 100644 --- a/CPP/7zip/var_clang_x86.mak +++ b/CPP/7zip/var_clang_x86.mak @@ -9,4 +9,3 @@ USE_ASM=1 CC=$(CROSS_COMPILE)clang CXX=$(CROSS_COMPILE)clang++ USE_CLANG=1 - diff --git a/CPP/7zip/var_gcc_arm.mak b/CPP/7zip/var_gcc_arm.mak new file mode 100644 index 0000000..7527e40 --- /dev/null +++ b/CPP/7zip/var_gcc_arm.mak @@ -0,0 +1,34 @@ +PLATFORM=arm +O=b/g_$(PLATFORM) +IS_X64= +IS_X86= +IS_ARM64= + +CROSS_COMPILE_ABI= +CROSS_COMPILE_ABI=arm-linux-musleabi +CROSS_COMPILE_ABI=arm-linux-musleabihf +CROSS_COMPILE_ABI=aarch64-linux-musl +CROSS_COMPILE_ABI=arm-linux-gnueabi +CROSS_COMPILE_ABI=arm-linux-gnueabihf + +COMPILER_VER_POSTFIX=-12 +COMPILER_VER_POSTFIX= + +CROSS_COMPILE_PREFIX= + +CROSS_COMPILE=$(CROSS_COMPILE_PREFIX)$(CROSS_COMPILE_ABI)- +CROSS_COMPILE= + +MY_ARCH= +MY_ARCH=-mtune=cortex-a53 -march=armv7-a +MY_ARCH=-mtune=cortex-a53 -march=armv4 +MY_ARCH=-mtune=cortex-a53 + +USE_ASM= + +LDFLAGS_STATIC_3=-static +LDFLAGS_STATIC_3= + +CC=$(CROSS_COMPILE)gcc$(COMPILER_VER_POSTFIX) +CXX=$(CROSS_COMPILE)g++$(COMPILER_VER_POSTFIX) +# -marm -march=armv5t -march=armv6 -march=armv7-a -march=armv8-a -march=armv8-a+crc+crypto diff --git a/CPP/7zip/var_gcc_x64.mak b/CPP/7zip/var_gcc_x64.mak index 1acf604..aeea203 100644 --- a/CPP/7zip/var_gcc_x64.mak +++ b/CPP/7zip/var_gcc_x64.mak @@ -4,6 +4,7 @@ IS_X64=1 IS_X86= IS_ARM64= CROSS_COMPILE= +MY_ARCH=-mavx512f -mavx512vl MY_ARCH= USE_ASM=1 CC=$(CROSS_COMPILE)gcc diff --git a/CPP/7zip/var_gcc_x86.mak b/CPP/7zip/var_gcc_x86.mak index 288bf94..f0718ec 100644 --- a/CPP/7zip/var_gcc_x86.mak +++ b/CPP/7zip/var_gcc_x86.mak @@ -8,4 +8,3 @@ MY_ARCH=-m32 USE_ASM=1 CC=$(CROSS_COMPILE)gcc CXX=$(CROSS_COMPILE)g++ - diff --git a/CPP/7zip/var_mac_arm64.mak b/CPP/7zip/var_mac_arm64.mak index adf5fa1..746e6db 100644 --- a/CPP/7zip/var_mac_arm64.mak +++ b/CPP/7zip/var_mac_arm64.mak @@ -4,6 +4,8 @@ IS_X64= IS_X86= IS_ARM64=1 CROSS_COMPILE= +#use this code to reduce features +MY_ARCH=-arch arm64 -march=armv8-a MY_ARCH=-arch arm64 USE_ASM=1 CC=$(CROSS_COMPILE)clang diff --git a/CPP/7zip/warn_gcc.mak b/CPP/7zip/warn_gcc.mak index 7eb1f57..b6ed9c3 100644 --- a/CPP/7zip/warn_gcc.mak +++ b/CPP/7zip/warn_gcc.mak @@ -38,7 +38,8 @@ CFLAGS_WARN_GCC_PPMD_UNALIGNED = \ -Wno-strict-aliasing \ -# CFLAGS_WARN = $(CFLAGS_WARN_GCC_4_8) +CFLAGS_WARN = $(CFLAGS_WARN_GCC_4_8) +CFLAGS_WARN = $(CFLAGS_WARN_GCC_6) CFLAGS_WARN = $(CFLAGS_WARN_GCC_9) # CXX_STD_FLAGS = -std=c++11 diff --git a/CPP/Build.mak b/CPP/Build.mak index 393fa2c..afb7ae8 100644 --- a/CPP/Build.mak +++ b/CPP/Build.mak @@ -1,7 +1,8 @@ LIBS = $(LIBS) oleaut32.lib ole32.lib +# CFLAGS = $(CFLAGS) -DZ7_NO_UNICODE !IFNDEF MY_NO_UNICODE -CFLAGS = $(CFLAGS) -DUNICODE -D_UNICODE +# CFLAGS = $(CFLAGS) -DUNICODE -D_UNICODE !ENDIF !IF "$(CC)" != "clang-cl" @@ -21,10 +22,14 @@ O=o # CFLAGS = $(CFLAGS) -FAsc -Fa$O/asm/ !ENDIF +# LFLAGS = $(LFLAGS) /guard:cf + !IF "$(PLATFORM)" == "x64" MY_ML = ml64 -WX #-Dx64 +!ELSEIF "$(PLATFORM)" == "arm64" +MY_ML = armasm64 !ELSEIF "$(PLATFORM)" == "arm" MY_ML = armasm -WX !ELSE @@ -52,6 +57,8 @@ LIBS = $(LIBS) user32.lib advapi32.lib shell32.lib !IF "$(PLATFORM)" == "arm" COMPL_ASM = $(MY_ML) $** $O/$(*B).obj +!ELSEIF "$(PLATFORM)" == "arm64" +COMPL_ASM = $(MY_ML) $** $O/$(*B).obj !ELSE COMPL_ASM = $(MY_ML) -c -Fo$O/ $** !ENDIF @@ -75,7 +82,8 @@ CFLAGS = $(CFLAGS) \ !ENDIF -!IFDEF MY_DYNAMIC_LINK +# !IFDEF MY_DYNAMIC_LINK +!IF "$(MY_DYNAMIC_LINK)" != "" CFLAGS = $(CFLAGS) -MD !ELSE !IFNDEF MY_SINGLE_THREAD @@ -169,6 +177,15 @@ LFLAGS = $(LFLAGS) /SUBSYSTEM:windows,$(MY_SUB_SYS_VER) !ENDIF +!IF "$(PLATFORM)" == "arm64" +CLANG_FLAGS_TARGET = --target=arm64-pc-windows-msvc +!ENDIF + +COMPL_CLANG_SPEC=clang-cl $(CLANG_FLAGS_TARGET) +COMPL_ASM_CLANG = $(COMPL_CLANG_SPEC) -nologo -c -Fo$O/ $(CFLAGS_WARN_LEVEL) -WX $** +# COMPL_C_CLANG = $(COMPL_CLANG_SPEC) $(CFLAGS_O2) + + PROGPATH = $O\$(PROG) COMPL_O1 = $(CC) $(CFLAGS_O1) $** @@ -224,6 +241,6 @@ predef: empty.c predef2: A.cpp $(COMPL) -EP -Zc:preprocessor -PD predef3: A.cpp - $(COMPL) -E -dM + $(COMPL) -E -dM predef4: A.cpp $(COMPL_O2) -E diff --git a/CPP/Common/AutoPtr.h b/CPP/Common/AutoPtr.h index 0be8a7a..e3c5763 100644 --- a/CPP/Common/AutoPtr.h +++ b/CPP/Common/AutoPtr.h @@ -3,20 +3,32 @@ #ifndef ZIP7_INC_COMMON_AUTOPTR_H #define ZIP7_INC_COMMON_AUTOPTR_H -template class CMyAutoPtr +template class CMyUniquePtr +// CMyAutoPtr { T *_p; -public: - CMyAutoPtr(T *p = NULL) : _p(p) {} - CMyAutoPtr(CMyAutoPtr& p): _p(p.release()) {} - CMyAutoPtr& operator=(CMyAutoPtr& p) + + CMyUniquePtr(CMyUniquePtr& p); // : _p(p.release()) {} + CMyUniquePtr& operator=(T *p); + CMyUniquePtr& operator=(CMyUniquePtr& p); + /* { reset(p.release()); return (*this); } - ~CMyAutoPtr() { delete _p; } + */ + void reset(T* p = NULL) + { + if (p != _p) + delete _p; + _p = p; + } +public: + CMyUniquePtr(T *p = NULL) : _p(p) {} + ~CMyUniquePtr() { delete _p; } T& operator*() const { return *_p; } - // T* operator->() const { return (&**this); } + T* operator->() const { return _p; } + // operator bool() const { return _p != NULL; } T* get() const { return _p; } T* release() { @@ -24,11 +36,10 @@ template class CMyAutoPtr _p = NULL; return tmp; } - void reset(T* p = NULL) + void Create_if_Empty() { - if (p != _p) - delete _p; - _p = p; + if (!_p) + _p = new T; } }; diff --git a/CPP/Common/CommandLineParser.cpp b/CPP/Common/CommandLineParser.cpp index 465e0fd..319f27f 100644 --- a/CPP/Common/CommandLineParser.cpp +++ b/CPP/Common/CommandLineParser.cpp @@ -6,6 +6,8 @@ namespace NCommandLineParser { +#ifdef _WIN32 + bool SplitCommandLine(const UString &src, UString &dest1, UString &dest2) { dest1.Empty(); @@ -14,7 +16,7 @@ bool SplitCommandLine(const UString &src, UString &dest1, UString &dest2) unsigned i; for (i = 0; i < src.Len(); i++) { - wchar_t c = src[i]; + const wchar_t c = src[i]; if ((c == L' ' || c == L'\t') && !quoteMode) { dest2 = src.Ptr(i + 1); @@ -30,6 +32,34 @@ bool SplitCommandLine(const UString &src, UString &dest1, UString &dest2) void SplitCommandLine(const UString &s, UStringVector &parts) { +#if 0 +/* we don't use CommandLineToArgvW() because + it can remove tail backslash: + "1\" + converted to + 1" +*/ + parts.Clear(); + { + int nArgs; + LPWSTR *szArgList = CommandLineToArgvW(s, &nArgs); + if (szArgList) + { + for (int i = 0; i < nArgs; i++) + { + // printf("%2d: |%S|\n", i, szArglist[i]); + parts.Add(szArgList[i]); + } + LocalFree(szArgList); + return; + } + } +#endif +/* +#ifdef _UNICODE + throw 20240406; +#else +*/ UString sTemp (s); sTemp.Trim(); parts.Clear(); @@ -42,7 +72,9 @@ void SplitCommandLine(const UString &s, UStringVector &parts) break; sTemp = s2; } +// #endif } +#endif static const char * const kStopSwitchParsing = "--"; @@ -78,7 +110,7 @@ bool CParser::ParseString(const UString &s, const CSwitchForm *switchForms, unsi for (unsigned i = 0; i < numSwitches; i++) { const char * const key = switchForms[i].Key; - unsigned switchLen = MyStringLen(key); + const unsigned switchLen = MyStringLen(key); if ((int)switchLen <= maxLen || pos + switchLen > s.Len()) continue; if (IsString1PrefixedByString2_NoCase_Ascii((const wchar_t *)s + pos, key)) @@ -133,7 +165,7 @@ bool CParser::ParseString(const UString &s, const CSwitchForm *switchForms, unsi case NSwitchType::kChar: if (rem == 1) { - wchar_t c = s[pos]; + const wchar_t c = s[pos]; if (c <= 0x7F) { sw.PostCharIndex = FindCharPosInString(form.PostCharSet, (char)c); @@ -150,6 +182,8 @@ bool CParser::ParseString(const UString &s, const CSwitchForm *switchForms, unsi sw.PostStrings.Add(s.Ptr(pos)); return true; } + // case NSwitchType::kSimple: + default: break; } if (pos != s.Len()) diff --git a/CPP/Common/Common.h b/CPP/Common/Common.h index 0c77ab4..cde0c38 100644 --- a/CPP/Common/Common.h +++ b/CPP/Common/Common.h @@ -7,7 +7,9 @@ #ifndef ZIP7_INC_COMMON_H #define ZIP7_INC_COMMON_H -#include "../../C/Compiler.h" +#include "../../C/Precomp.h" +#include "Common0.h" +#include "MyWindows.h" /* This file is included to all cpp files in 7-Zip. @@ -17,297 +19,10 @@ So 7-Zip includes "Common.h" in both modes: and without precompiled StdAfx.h -If you use 7-Zip code, you must include "Common.h" before other h files of 7-zip. -If you don't need some things that are used in 7-Zip, -you can change this h file or h files included in this file. -*/ - -#ifdef _MSC_VER - #pragma warning(disable : 4710) // function not inlined - // 'CUncopyable::CUncopyable': - #pragma warning(disable : 4514) // unreferenced inline function has been removed - #if _MSC_VER < 1300 - #pragma warning(disable : 4702) // unreachable code - #pragma warning(disable : 4714) // function marked as __forceinline not inlined - #pragma warning(disable : 4786) // identifier was truncated to '255' characters in the debug information - #endif - #if _MSC_VER < 1400 - #pragma warning(disable : 4511) // copy constructor could not be generated // #pragma warning(disable : 4512) // assignment operator could not be generated - #pragma warning(disable : 4512) // assignment operator could not be generated - #endif - #if _MSC_VER > 1400 && _MSC_VER <= 1900 - // #pragma warning(disable : 4996) - // strcat: This function or variable may be unsafe - // GetVersion was declared deprecated - #endif - -#if _MSC_VER > 1200 -// -Wall warnings - -#if _MSC_VER <= 1600 -#pragma warning(disable : 4917) // 'OLE_HANDLE' : a GUID can only be associated with a class, interface or namespace -#endif - -// #pragma warning(disable : 4061) // enumerator '' in switch of enum '' is not explicitly handled by a case label -// #pragma warning(disable : 4266) // no override available for virtual member function from base ''; function is hidden -#pragma warning(disable : 4625) // copy constructor was implicitly defined as deleted -#pragma warning(disable : 4626) // assignment operator was implicitly defined as deleted -#if _MSC_VER >= 1600 && _MSC_VER < 1920 -#pragma warning(disable : 4571) // Informational: catch(...) semantics changed since Visual C++ 7.1; structured exceptions (SEH) are no longer caught -#endif -#if _MSC_VER >= 1600 -#pragma warning(disable : 4365) // 'initializing' : conversion from 'int' to 'unsigned int', signed / unsigned mismatch -#endif -#if _MSC_VER < 1800 -// we disable the warning, if we don't use 'final' in class -#pragma warning(disable : 4265) // class has virtual functions, but destructor is not virtual -#endif - -#if _MSC_VER >= 1900 -#pragma warning(disable : 5026) // move constructor was implicitly defined as deleted -#pragma warning(disable : 5027) // move assignment operator was implicitly defined as deleted -#endif -#if _MSC_VER >= 1912 -#pragma warning(disable : 5039) // pointer or reference to potentially throwing function passed to 'extern "C"' function under - EHc.Undefined behavior may occur if this function throws an exception. -#endif -#if _MSC_VER >= 1925 -// #pragma warning(disable : 5204) // 'ISequentialInStream' : class has virtual functions, but its trivial destructor is not virtual; instances of objects derived from this class may not be destructed correctly -#endif -#if _MSC_VER >= 1934 -// #pragma warning(disable : 5264) // const variable is not used -#endif - -#endif // _MSC_VER > 1200 -#endif // _MSC_VER - - -#if defined(_MSC_VER) // && !defined(__clang__) -#define Z7_DECLSPEC_NOTHROW __declspec(nothrow) -#elif defined(__clang__) || defined(__GNUC__) -#define Z7_DECLSPEC_NOTHROW __attribute__((nothrow)) -#else -#define Z7_DECLSPEC_NOTHROW -#endif - -/* -#if defined (_MSC_VER) && _MSC_VER >= 1900 \ - || defined(__clang__) && __clang_major__ >= 6 \ - || defined(__GNUC__) && __GNUC__ >= 6 - #define Z7_noexcept noexcept -#else - #define Z7_noexcept throw() -#endif -*/ - - -#if defined(__clang__) - -// noexcept, final, = delete -#pragma GCC diagnostic ignored "-Wc++98-compat" -#if __clang_major__ >= 4 -// throw() dynamic exception specifications are deprecated -#pragma GCC diagnostic ignored "-Wdeprecated-dynamic-exception-spec" -#endif -#pragma GCC diagnostic ignored "-Wold-style-cast" -#pragma GCC diagnostic ignored "-Wglobal-constructors" -#pragma GCC diagnostic ignored "-Wexit-time-destructors" - -// #pragma GCC diagnostic ignored "-Wunused-private-field" -// #pragma GCC diagnostic ignored "-Wnonportable-system-include-path" -// #pragma GCC diagnostic ignored "-Wsuggest-override" -// #pragma GCC diagnostic ignored "-Wsign-conversion" -// #pragma GCC diagnostic ignored "-Winconsistent-missing-override" -// #pragma GCC diagnostic ignored "-Wsuggest-destructor-override" -// #pragma GCC diagnostic ignored "-Wnon-virtual-dtor" -// #pragma GCC diagnostic ignored "-Wdeprecated-copy-with-user-provided-dtor" -// #pragma GCC diagnostic ignored "-Wdeprecated-copy-dtor" -// #ifndef _WIN32 -// #pragma GCC diagnostic ignored "-Wweak-vtables" -// #endif -/* -#if defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 40400) \ - || defined(Z7_CLANG_VERSION) && (Z7_CLANG_VERSION >= 30000) -// enumeration values not explicitly handled in switch -#pragma GCC diagnostic ignored "-Wswitch-enum" -#endif -*/ -#endif // __clang__ - - -#ifdef __GNUC__ -// #pragma GCC diagnostic ignored "-Wdelete-non-virtual-dtor" -#endif - - -/* There is BUG in MSVC 6.0 compiler for operator new[]: - It doesn't check overflow, when it calculates size in bytes for allocated array. - So we can use Z7_ARRAY_NEW macro instead of new[] operator. */ - -#if defined(_MSC_VER) && (_MSC_VER == 1200) && !defined(_WIN64) - #define Z7_ARRAY_NEW(p, T, size) p = new T[((size) > (unsigned)0xFFFFFFFF / sizeof(T)) ? (unsigned)0xFFFFFFFF / sizeof(T) : (size)]; -#else - #define Z7_ARRAY_NEW(p, T, size) p = new T[size]; -#endif - -#if (defined(__GNUC__) && (__GNUC__ >= 8)) - #define Z7_ATTR_NORETURN __attribute__((noreturn)) -#elif (defined(__clang__) && (__clang_major__ >= 3)) - #if __has_feature(cxx_attributes) - #define Z7_ATTR_NORETURN [[noreturn]] - #else - #define Z7_ATTR_NORETURN __attribute__((noreturn)) - #endif -#elif (defined(_MSC_VER) && (_MSC_VER >= 1900)) - #define Z7_ATTR_NORETURN [[noreturn]] -#else - #define Z7_ATTR_NORETURN -#endif - - -// final in "GCC 4.7.0" -// In C++98 and C++03 code the alternative spelling __final can be used instead (this is a GCC extension.) - -#if defined (__cplusplus) && __cplusplus >= 201103L \ - || defined(_MSC_VER) && _MSC_VER >= 1800 \ - || defined(__clang__) && __clang_major__ >= 4 \ - /* || defined(__GNUC__) && __GNUC__ >= 9 */ - #define Z7_final final - #if defined(__clang__) && __cplusplus < 201103L - #pragma GCC diagnostic ignored "-Wc++11-extensions" - #endif -#elif defined (__cplusplus) && __cplusplus >= 199711L \ - && defined(__GNUC__) && __GNUC__ >= 4 && !defined(__clang__) - #define Z7_final __final -#else - #define Z7_final - #if defined(__clang__) && __clang_major__ >= 4 \ - || defined(__GNUC__) && __GNUC__ >= 4 - #pragma GCC diagnostic ignored "-Wnon-virtual-dtor" - #pragma GCC diagnostic ignored "-Wdelete-non-virtual-dtor" - #endif -#endif - -#define Z7_class_final(c) class c Z7_final - - -#if defined (__cplusplus) && __cplusplus >= 201103L \ - || (defined(_MSC_VER) && _MSC_VER >= 1800) - #define Z7_CPP_IS_SUPPORTED_default - #define Z7_eq_delete = delete - // #define Z7_DECL_DEFAULT_COPY_CONSTRUCTOR_IF_SUPPORTED(c) c(const c& k) = default; -#else - #define Z7_eq_delete - // #define Z7_DECL_DEFAULT_COPY_CONSTRUCTOR_IF_SUPPORTED(c) -#endif - - -#if defined(__cplusplus) && (__cplusplus >= 201103L) \ - || defined(_MSC_VER) && (_MSC_VER >= 1400) /* && (_MSC_VER != 1600) */ \ - || defined(__clang__) && __clang_major__ >= 4 - #if defined(_MSC_VER) && (_MSC_VER == 1600) /* && (_MSC_VER != 1600) */ - #pragma warning(disable : 4481) // nonstandard extension used: override specifier 'override' - #define Z7_DESTRUCTOR_override - #else - #define Z7_DESTRUCTOR_override override - #endif - #define Z7_override override -#else - #define Z7_override - #define Z7_DESTRUCTOR_override -#endif - - - -#define Z7_CLASS_NO_COPY(cls) \ - private: \ - cls(const cls &) Z7_eq_delete; \ - cls &operator=(const cls &) Z7_eq_delete; - -class CUncopyable -{ -protected: - CUncopyable() {} // allow constructor - // ~CUncopyable() {} - Z7_CLASS_NO_COPY(CUncopyable) -}; - -#define MY_UNCOPYABLE :private CUncopyable -// #define MY_UNCOPYABLE - - -typedef void (*Z7_void_Function)(void); - -#if defined(__clang__) || defined(__GNUC__) -#define Z7_CAST_FUNC(t, e) reinterpret_cast(reinterpret_cast(e)) -#else -#define Z7_CAST_FUNC(t, e) reinterpret_cast(reinterpret_cast(e)) -// #define Z7_CAST_FUNC(t, e) reinterpret_cast(e) -#endif - -#define Z7_GET_PROC_ADDRESS(func_type, hmodule, func_name) \ - Z7_CAST_FUNC(func_type, GetProcAddress(hmodule, func_name)) - -// || defined(__clang__) -// || defined(__GNUC__) - -#if defined(_MSC_VER) && (_MSC_VER >= 1400) -#define Z7_DECLSPEC_NOVTABLE __declspec(novtable) -#else -#define Z7_DECLSPEC_NOVTABLE -#endif - -#ifdef __clang__ -#define Z7_PURE_INTERFACES_BEGIN \ -_Pragma("GCC diagnostic push") \ -_Pragma("GCC diagnostic ignored \"-Wnon-virtual-dtor\"") -_Pragma("GCC diagnostic ignored \"-Wweak-vtables\"") -#define Z7_PURE_INTERFACES_END \ -_Pragma("GCC diagnostic pop") -#else -#define Z7_PURE_INTERFACES_BEGIN -#define Z7_PURE_INTERFACES_END -#endif - -// NewHandler.h and NewHandler.cpp redefine operator new() to throw exceptions, if compiled with old MSVC compilers -#include "NewHandler.h" - -/* -// #define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) -#ifndef ARRAY_SIZE -#define ARRAY_SIZE(a) Z7_ARRAY_SIZE(a) -#endif +include "Common.h" before other h files of 7-zip, + if you need predefined macros. +do not include "Common.h", if you need only interfaces, + and you don't need predefined macros. */ -#endif // ZIP7_INC_COMMON_H - - - -// #define Z7_REDEFINE_NULL - -#if defined(Z7_REDEFINE_NULL) /* && (!defined(__clang__) || defined(_MSC_VER)) */ - -// NULL is defined in -#include -#undef NULL - -#ifdef __cplusplus - #if defined (__cplusplus) && __cplusplus >= 201103L \ - || (defined(_MSC_VER) && _MSC_VER >= 1800) - #define NULL nullptr - #else - #define NULL 0 - #endif -#else - #define NULL ((void *)0) -#endif - -#else // Z7_REDEFINE_NULL - -#if defined(__clang__) && __clang_major__ >= 5 -#pragma GCC diagnostic ignored "-Wzero-as-null-pointer-constant" #endif - -#endif // Z7_REDEFINE_NULL - -// for precompiler: -#include "MyWindows.h" diff --git a/CPP/Common/Common0.h b/CPP/Common/Common0.h new file mode 100644 index 0000000..55606cd --- /dev/null +++ b/CPP/Common/Common0.h @@ -0,0 +1,330 @@ +// Common0.h + +#if defined(_MSC_VER) && _MSC_VER >= 1800 +#pragma warning(disable : 4464) // relative include path contains '..' +#endif + +#ifndef ZIP7_INC_COMMON0_H +#define ZIP7_INC_COMMON0_H + +#include "../../C/Compiler.h" + +/* +This file contains compiler related things for cpp files. +This file is included to all cpp files in 7-Zip via "Common.h". +Also this file is included in "IDecl.h" (that is included in interface files). +So external modules can use 7-Zip interfaces without +predefined macros defined in "Common.h". +*/ + +#ifdef _MSC_VER + #pragma warning(disable : 4710) // function not inlined + // 'CUncopyable::CUncopyable': + #pragma warning(disable : 4514) // unreferenced inline function has been removed + #if _MSC_VER < 1300 + #pragma warning(disable : 4702) // unreachable code + #pragma warning(disable : 4714) // function marked as __forceinline not inlined + #pragma warning(disable : 4786) // identifier was truncated to '255' characters in the debug information + #endif + #if _MSC_VER < 1400 + #pragma warning(disable : 4511) // copy constructor could not be generated // #pragma warning(disable : 4512) // assignment operator could not be generated + #pragma warning(disable : 4512) // assignment operator could not be generated + #endif + #if _MSC_VER > 1400 && _MSC_VER <= 1900 + // #pragma warning(disable : 4996) + // strcat: This function or variable may be unsafe + // GetVersion was declared deprecated + #endif + +#if _MSC_VER > 1200 +// -Wall warnings + +#if _MSC_VER <= 1600 +#pragma warning(disable : 4917) // 'OLE_HANDLE' : a GUID can only be associated with a class, interface or namespace +#endif + +// #pragma warning(disable : 4061) // enumerator '' in switch of enum '' is not explicitly handled by a case label +// #pragma warning(disable : 4266) // no override available for virtual member function from base ''; function is hidden +#pragma warning(disable : 4625) // copy constructor was implicitly defined as deleted +#pragma warning(disable : 4626) // assignment operator was implicitly defined as deleted +#if _MSC_VER >= 1600 && _MSC_VER < 1920 +#pragma warning(disable : 4571) // Informational: catch(...) semantics changed since Visual C++ 7.1; structured exceptions (SEH) are no longer caught +#endif +#if _MSC_VER >= 1600 +#pragma warning(disable : 4365) // 'initializing' : conversion from 'int' to 'unsigned int', signed / unsigned mismatch +#endif +#if _MSC_VER < 1800 +// we disable the warning, if we don't use 'final' in class +#pragma warning(disable : 4265) // class has virtual functions, but destructor is not virtual +#endif + +#if _MSC_VER >= 1900 +#pragma warning(disable : 5026) // move constructor was implicitly defined as deleted +#pragma warning(disable : 5027) // move assignment operator was implicitly defined as deleted +#endif +#if _MSC_VER >= 1912 +#pragma warning(disable : 5039) // pointer or reference to potentially throwing function passed to 'extern "C"' function under - EHc.Undefined behavior may occur if this function throws an exception. +#endif +#if _MSC_VER >= 1925 +// #pragma warning(disable : 5204) // 'ISequentialInStream' : class has virtual functions, but its trivial destructor is not virtual; instances of objects derived from this class may not be destructed correctly +#endif +#if _MSC_VER >= 1934 +// #pragma warning(disable : 5264) // const variable is not used +#endif + +#endif // _MSC_VER > 1200 +#endif // _MSC_VER + + +#if defined(_MSC_VER) // && !defined(__clang__) +#define Z7_DECLSPEC_NOTHROW __declspec(nothrow) +#elif defined(__clang__) || defined(__GNUC__) +#define Z7_DECLSPEC_NOTHROW __attribute__((nothrow)) +#else +#define Z7_DECLSPEC_NOTHROW +#endif + +/* +#if defined (_MSC_VER) && _MSC_VER >= 1900 \ + || defined(__clang__) && __clang_major__ >= 6 \ + || defined(__GNUC__) && __GNUC__ >= 6 + #define Z7_noexcept noexcept +#else + #define Z7_noexcept throw() +#endif +*/ + + +#if defined(__clang__) + +#if /* defined(_WIN32) && */ __clang_major__ >= 16 +#pragma GCC diagnostic ignored "-Wc++98-compat-pedantic" +#endif + +#if __clang_major__ >= 4 && __clang_major__ < 12 && !defined(_WIN32) +/* +if compiled with new GCC libstdc++, GCC libstdc++ can use: +13.2.0/include/c++/ + : #define _NEW + : #define _GLIBCXX_STDLIB_H 1 +*/ +#pragma GCC diagnostic ignored "-Wreserved-id-macro" +#endif + +// noexcept, final, = delete +#pragma GCC diagnostic ignored "-Wc++98-compat" +#if __clang_major__ >= 4 +// throw() dynamic exception specifications are deprecated +#pragma GCC diagnostic ignored "-Wdeprecated-dynamic-exception-spec" +#endif + +#if __clang_major__ <= 6 // check it +#pragma GCC diagnostic ignored "-Wsign-conversion" +#endif + +#pragma GCC diagnostic ignored "-Wold-style-cast" +#pragma GCC diagnostic ignored "-Wglobal-constructors" +#pragma GCC diagnostic ignored "-Wexit-time-destructors" + +#if defined(Z7_LLVM_CLANG_VERSION) && __clang_major__ >= 18 // 18.1.0RC +#pragma GCC diagnostic ignored "-Wswitch-default" +#endif +// #pragma GCC diagnostic ignored "-Wunused-private-field" +// #pragma GCC diagnostic ignored "-Wnonportable-system-include-path" +// #pragma GCC diagnostic ignored "-Wsuggest-override" +// #pragma GCC diagnostic ignored "-Wsign-conversion" +// #pragma GCC diagnostic ignored "-Winconsistent-missing-override" +// #pragma GCC diagnostic ignored "-Wsuggest-destructor-override" +// #pragma GCC diagnostic ignored "-Wnon-virtual-dtor" +// #pragma GCC diagnostic ignored "-Wdeprecated-copy-with-user-provided-dtor" +// #pragma GCC diagnostic ignored "-Wdeprecated-copy-dtor" +// #ifndef _WIN32 +// #pragma GCC diagnostic ignored "-Wweak-vtables" +// #endif +/* +#if defined(Z7_GCC_VERSION) && (Z7_GCC_VERSION >= 40400) \ + || defined(Z7_CLANG_VERSION) && (Z7_CLANG_VERSION >= 30000) +// enumeration values not explicitly handled in switch +#pragma GCC diagnostic ignored "-Wswitch-enum" +#endif +*/ +#endif // __clang__ + + +#ifdef __GNUC__ +// #pragma GCC diagnostic ignored "-Wdelete-non-virtual-dtor" +#endif + + +/* There is BUG in MSVC 6.0 compiler for operator new[]: + It doesn't check overflow, when it calculates size in bytes for allocated array. + So we can use Z7_ARRAY_NEW macro instead of new[] operator. */ + +#if defined(_MSC_VER) && (_MSC_VER == 1200) && !defined(_WIN64) + #define Z7_ARRAY_NEW(p, T, size) p = new T[((size) > 0xFFFFFFFFu / sizeof(T)) ? 0xFFFFFFFFu / sizeof(T) : (size)]; +#else + #define Z7_ARRAY_NEW(p, T, size) p = new T[size]; +#endif + +#if (defined(__GNUC__) && (__GNUC__ >= 8)) + #define Z7_ATTR_NORETURN __attribute__((noreturn)) +#elif (defined(__clang__) && (__clang_major__ >= 3)) + #if __has_feature(cxx_attributes) + #define Z7_ATTR_NORETURN [[noreturn]] + #else + #define Z7_ATTR_NORETURN __attribute__((noreturn)) + #endif +#elif (defined(_MSC_VER) && (_MSC_VER >= 1900)) + #define Z7_ATTR_NORETURN [[noreturn]] +#else + #define Z7_ATTR_NORETURN +#endif + + +// final in "GCC 4.7.0" +// In C++98 and C++03 code the alternative spelling __final can be used instead (this is a GCC extension.) + +#if defined (__cplusplus) && __cplusplus >= 201103L \ + || defined(_MSC_VER) && _MSC_VER >= 1800 \ + || defined(__clang__) && __clang_major__ >= 4 \ + /* || defined(__GNUC__) && __GNUC__ >= 9 */ + #define Z7_final final + #if defined(__clang__) && __cplusplus < 201103L + #pragma GCC diagnostic ignored "-Wc++11-extensions" + #endif +#elif defined (__cplusplus) && __cplusplus >= 199711L \ + && defined(__GNUC__) && __GNUC__ >= 4 && !defined(__clang__) + #define Z7_final __final +#else + #define Z7_final + #if defined(__clang__) && __clang_major__ >= 4 \ + || defined(__GNUC__) && __GNUC__ >= 4 + #pragma GCC diagnostic ignored "-Wnon-virtual-dtor" + #pragma GCC diagnostic ignored "-Wdelete-non-virtual-dtor" + #endif +#endif + +#define Z7_class_final(c) class c Z7_final + + +#if defined (__cplusplus) && __cplusplus >= 201103L \ + || (defined(_MSC_VER) && _MSC_VER >= 1800) + #define Z7_CPP_IS_SUPPORTED_default + #define Z7_eq_delete = delete + // #define Z7_DECL_DEFAULT_COPY_CONSTRUCTOR_IF_SUPPORTED(c) c(const c& k) = default; +#else + #define Z7_eq_delete + // #define Z7_DECL_DEFAULT_COPY_CONSTRUCTOR_IF_SUPPORTED(c) +#endif + + +#if defined(__cplusplus) && (__cplusplus >= 201103L) \ + || defined(_MSC_VER) && (_MSC_VER >= 1400) /* && (_MSC_VER != 1600) */ \ + || defined(__clang__) && __clang_major__ >= 4 + #if defined(_MSC_VER) && (_MSC_VER == 1600) /* && (_MSC_VER != 1600) */ + #pragma warning(disable : 4481) // nonstandard extension used: override specifier 'override' + #define Z7_DESTRUCTOR_override + #else + #define Z7_DESTRUCTOR_override override + #endif + #define Z7_override override +#else + #define Z7_override + #define Z7_DESTRUCTOR_override +#endif + + + +#define Z7_CLASS_NO_COPY(cls) \ + private: \ + cls(const cls &) Z7_eq_delete; \ + cls &operator=(const cls &) Z7_eq_delete; + +class CUncopyable +{ +protected: + CUncopyable() {} // allow constructor + // ~CUncopyable() {} + Z7_CLASS_NO_COPY(CUncopyable) +}; + +#define MY_UNCOPYABLE :private CUncopyable +// #define MY_UNCOPYABLE + + +// typedef void (*Z7_void_Function)(void); + +#if defined(__clang__) || defined(__GNUC__) +#define Z7_CAST_FUNC(t, e) reinterpret_cast(reinterpret_cast(e)) +#else +#define Z7_CAST_FUNC(t, e) reinterpret_cast(reinterpret_cast(e)) +// #define Z7_CAST_FUNC(t, e) reinterpret_cast(e) +#endif + +#define Z7_GET_PROC_ADDRESS(func_type, hmodule, func_name) \ + Z7_CAST_FUNC(func_type, GetProcAddress(hmodule, func_name)) + +// || defined(__clang__) +// || defined(__GNUC__) + +#if defined(_MSC_VER) && (_MSC_VER >= 1400) +#define Z7_DECLSPEC_NOVTABLE __declspec(novtable) +#else +#define Z7_DECLSPEC_NOVTABLE +#endif + +#ifdef __clang__ +#define Z7_PURE_INTERFACES_BEGIN \ +_Pragma("GCC diagnostic push") \ +_Pragma("GCC diagnostic ignored \"-Wnon-virtual-dtor\"") +_Pragma("GCC diagnostic ignored \"-Wweak-vtables\"") +#define Z7_PURE_INTERFACES_END \ +_Pragma("GCC diagnostic pop") +#else +#define Z7_PURE_INTERFACES_BEGIN +#define Z7_PURE_INTERFACES_END +#endif + +// NewHandler.h and NewHandler.cpp redefine operator new() to throw exceptions, if compiled with old MSVC compilers +#include "NewHandler.h" + +/* +// #define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) +#ifndef ARRAY_SIZE +#define ARRAY_SIZE(a) Z7_ARRAY_SIZE(a) +#endif +*/ + +#endif // ZIP7_INC_COMMON0_H + + + +// #define Z7_REDEFINE_NULL + +#if defined(Z7_REDEFINE_NULL) /* && (!defined(__clang__) || defined(_MSC_VER)) */ + +// NULL is defined in +#include +#undef NULL + +#ifdef __cplusplus + #if defined (__cplusplus) && __cplusplus >= 201103L \ + || (defined(_MSC_VER) && _MSC_VER >= 1800) + #define NULL nullptr + #else + #define NULL 0 + #endif +#else + #define NULL ((void *)0) +#endif + +#else // Z7_REDEFINE_NULL + +#if defined(__clang__) && __clang_major__ >= 5 +#pragma GCC diagnostic ignored "-Wzero-as-null-pointer-constant" +#endif + +#endif // Z7_REDEFINE_NULL + +// for precompiler: +// #include "MyWindows.h" diff --git a/CPP/Common/CrcReg.cpp b/CPP/Common/CrcReg.cpp index 6cda892..cd5d3da 100644 --- a/CPP/Common/CrcReg.cpp +++ b/CPP/Common/CrcReg.cpp @@ -11,14 +11,6 @@ EXTERN_C_BEGIN -// UInt32 Z7_FASTCALL CrcUpdateT1(UInt32 v, const void *data, size_t size, const UInt32 *table); - -extern CRC_FUNC g_CrcUpdate; -// extern CRC_FUNC g_CrcUpdateT4; -extern CRC_FUNC g_CrcUpdateT8; -extern CRC_FUNC g_CrcUpdateT0_32; -extern CRC_FUNC g_CrcUpdateT0_64; - EXTERN_C_END Z7_CLASS_IMP_COM_2( @@ -27,7 +19,7 @@ Z7_CLASS_IMP_COM_2( , ICompressSetCoderProperties ) UInt32 _crc; - CRC_FUNC _updateFunc; + Z7_CRC_UPDATE_FUNC _updateFunc; Z7_CLASS_NO_COPY(CCrcHasher) @@ -40,17 +32,10 @@ Z7_CLASS_IMP_COM_2( bool CCrcHasher::SetFunctions(UInt32 tSize) { - CRC_FUNC f = NULL; - if (tSize == 0) f = g_CrcUpdate; - // else if (tSize == 1) f = CrcUpdateT1; - // else if (tSize == 4) f = g_CrcUpdateT4; - else if (tSize == 8) f = g_CrcUpdateT8; - else if (tSize == 32) f = g_CrcUpdateT0_32; - else if (tSize == 64) f = g_CrcUpdateT0_64; - + const Z7_CRC_UPDATE_FUNC f = z7_GetFunc_CrcUpdate(tSize); if (!f) { - _updateFunc = g_CrcUpdate; + _updateFunc = CrcUpdate; return false; } _updateFunc = f; @@ -80,7 +65,7 @@ Z7_COM7F_IMF2(void, CCrcHasher::Init()) Z7_COM7F_IMF2(void, CCrcHasher::Update(const void *data, UInt32 size)) { - _crc = _updateFunc(_crc, data, size, g_CrcTable); + _crc = _updateFunc(_crc, data, size); } Z7_COM7F_IMF2(void, CCrcHasher::Final(Byte *digest)) diff --git a/CPP/Common/DynamicBuffer.h b/CPP/Common/DynamicBuffer.h index 714be4a..b03d371 100644 --- a/CPP/Common/DynamicBuffer.h +++ b/CPP/Common/DynamicBuffer.h @@ -5,7 +5,7 @@ #include -#include "Common.h" +#include "MyTypes.h" template class CDynamicBuffer { @@ -43,6 +43,14 @@ template class CDynamicBuffer operator const T *() const { return _items; } ~CDynamicBuffer() { delete []_items; } + void Free() + { + delete []_items; + _items = NULL; + _size = 0; + _pos = 0; + } + T *GetCurPtrAndGrow(size_t addSize) { size_t rem = _size - _pos; @@ -63,6 +71,6 @@ template class CDynamicBuffer // void Empty() { _pos = 0; } }; -typedef CDynamicBuffer CByteDynamicBuffer; +typedef CDynamicBuffer CByteDynamicBuffer; #endif diff --git a/CPP/Common/IntToString.cpp b/CPP/Common/IntToString.cpp index 21b0680..0a7cb3b 100644 --- a/CPP/Common/IntToString.cpp +++ b/CPP/Common/IntToString.cpp @@ -7,10 +7,19 @@ #include "IntToString.h" #define CONVERT_INT_TO_STR(charType, tempSize) \ - unsigned char temp[tempSize]; unsigned i = 0; \ - while (val >= 10) { temp[i++] = (unsigned char)('0' + (unsigned)(val % 10)); val /= 10; } \ - *s++ = (charType)('0' + (unsigned)val); \ - while (i != 0) { i--; *s++ = (charType)temp[i]; } \ + if (val < 10) \ + *s++ = (charType)('0' + (unsigned)val); \ + else { \ + Byte temp[tempSize]; \ + size_t i = 0; \ + do { \ + temp[++i] = (Byte)('0' + (unsigned)(val % 10)); \ + val /= 10; } \ + while (val >= 10); \ + *s++ = (charType)('0' + (unsigned)val); \ + do { *s++ = (charType)temp[i]; } \ + while (--i); \ + } \ *s = 0; \ return s; @@ -22,88 +31,109 @@ char * ConvertUInt32ToString(UInt32 val, char *s) throw() char * ConvertUInt64ToString(UInt64 val, char *s) throw() { if (val <= (UInt32)0xFFFFFFFF) - { return ConvertUInt32ToString((UInt32)val, s); - } CONVERT_INT_TO_STR(char, 24) } -void ConvertUInt64ToOct(UInt64 val, char *s) throw() +wchar_t * ConvertUInt32ToString(UInt32 val, wchar_t *s) throw() { - UInt64 v = val; - unsigned i; - for (i = 1;; i++) + CONVERT_INT_TO_STR(wchar_t, 16) +} + +wchar_t * ConvertUInt64ToString(UInt64 val, wchar_t *s) throw() +{ + if (val <= (UInt32)0xFFFFFFFF) + return ConvertUInt32ToString((UInt32)val, s); + CONVERT_INT_TO_STR(wchar_t, 24) +} + +void ConvertInt64ToString(Int64 val, char *s) throw() +{ + if (val < 0) { - v >>= 3; - if (v == 0) - break; + *s++ = '-'; + val = -val; } - s[i] = 0; - do + ConvertUInt64ToString((UInt64)val, s); +} + +void ConvertInt64ToString(Int64 val, wchar_t *s) throw() +{ + if (val < 0) { - unsigned t = (unsigned)(val & 0x7); - val >>= 3; - s[--i] = (char)('0' + t); + *s++ = L'-'; + val = -val; } - while (i); + ConvertUInt64ToString((UInt64)val, s); } -#define GET_HEX_CHAR(t) ((char)(((t < 10) ? ('0' + t) : ('A' + (t - 10))))) - -static inline char GetHexChar(unsigned t) { return GET_HEX_CHAR(t); } +void ConvertUInt64ToOct(UInt64 val, char *s) throw() +{ + { + UInt64 v = val; + do + s++; + while (v >>= 3); + } + *s = 0; + do + { + const unsigned t = (unsigned)val & 7; + *--s = (char)('0' + t); + } + while (val >>= 3); +} +MY_ALIGN(16) const char k_Hex_Upper[16] = + { '0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F' }; +MY_ALIGN(16) const char k_Hex_Lower[16] = + { '0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f' }; void ConvertUInt32ToHex(UInt32 val, char *s) throw() { - UInt32 v = val; - unsigned i; - for (i = 1;; i++) { - v >>= 4; - if (v == 0) - break; + UInt32 v = val; + do + s++; + while (v >>= 4); } - s[i] = 0; + *s = 0; do { - unsigned t = (unsigned)(val & 0xF); - val >>= 4; - s[--i] = GET_HEX_CHAR(t); + const unsigned t = (unsigned)val & 0xF; + *--s = GET_HEX_CHAR_UPPER(t); } - while (i); + while (val >>= 4); } - void ConvertUInt64ToHex(UInt64 val, char *s) throw() { - UInt64 v = val; - unsigned i; - for (i = 1;; i++) { - v >>= 4; - if (v == 0) - break; + UInt64 v = val; + do + s++; + while (v >>= 4); } - s[i] = 0; + *s = 0; do { - unsigned t = (unsigned)(val & 0xF); - val >>= 4; - s[--i] = GET_HEX_CHAR(t); + const unsigned t = (unsigned)val & 0xF; + *--s = GET_HEX_CHAR_UPPER(t); } - while (i); + while (val >>= 4); } void ConvertUInt32ToHex8Digits(UInt32 val, char *s) throw() { s[8] = 0; - for (int i = 7; i >= 0; i--) + int i = 7; + do { - unsigned t = val & 0xF; - val >>= 4; - s[i] = GET_HEX_CHAR(t); + { const unsigned t = (unsigned)val & 0xF; s[i--] = GET_HEX_CHAR_UPPER(t); } + { const unsigned t = (Byte)val >> 4; val >>= 8; s[i--] = GET_HEX_CHAR_UPPER(t); } } + while (i >= 0); } /* @@ -112,81 +142,74 @@ void ConvertUInt32ToHex8Digits(UInt32 val, wchar_t *s) s[8] = 0; for (int i = 7; i >= 0; i--) { - unsigned t = val & 0xF; + const unsigned t = (unsigned)val & 0xF; val >>= 4; - s[i] = (wchar_t)(((t < 10) ? ('0' + t) : ('A' + (t - 10)))); + s[i] = GET_HEX_CHAR(t); } } */ -wchar_t * ConvertUInt32ToString(UInt32 val, wchar_t *s) throw() -{ - CONVERT_INT_TO_STR(wchar_t, 16) -} - -wchar_t * ConvertUInt64ToString(UInt64 val, wchar_t *s) throw() -{ - if (val <= (UInt32)0xFFFFFFFF) - { - return ConvertUInt32ToString((UInt32)val, s); - } - CONVERT_INT_TO_STR(wchar_t, 24) -} -void ConvertInt64ToString(Int64 val, char *s) throw() -{ - if (val < 0) - { - *s++ = '-'; - val = -val; - } - ConvertUInt64ToString((UInt64)val, s); -} +MY_ALIGN(16) static const Byte k_Guid_Pos[] = + { 6,4,2,0, 11,9, 16,14, 19,21, 24,26,28,30,32,34 }; -void ConvertInt64ToString(Int64 val, wchar_t *s) throw() +char *RawLeGuidToString(const Byte *g, char *s) throw() { - if (val < 0) + s[ 8] = '-'; + s[13] = '-'; + s[18] = '-'; + s[23] = '-'; + s[36] = 0; + for (unsigned i = 0; i < 16; i++) { - *s++ = L'-'; - val = -val; + char *s2 = s + k_Guid_Pos[i]; + const unsigned v = g[i]; + s2[0] = GET_HEX_CHAR_UPPER(v >> 4); + s2[1] = GET_HEX_CHAR_UPPER(v & 0xF); } - ConvertUInt64ToString((UInt64)val, s); + return s + 36; } - -static void ConvertByteToHex2Digits(unsigned v, char *s) throw() +char *RawLeGuidToString_Braced(const Byte *g, char *s) throw() { - s[0] = GetHexChar(v >> 4); - s[1] = GetHexChar(v & 0xF); + *s++ = '{'; + s = RawLeGuidToString(g, s); + *s++ = '}'; + *s = 0; + return s; } -static void ConvertUInt16ToHex4Digits(UInt32 val, char *s) throw() -{ - ConvertByteToHex2Digits(val >> 8, s); - ConvertByteToHex2Digits(val & 0xFF, s + 2); -} -char *RawLeGuidToString(const Byte *g, char *s) throw() +void ConvertDataToHex_Lower(char *dest, const Byte *src, size_t size) throw() { - ConvertUInt32ToHex8Digits(GetUi32(g ), s); s += 8; *s++ = '-'; - ConvertUInt16ToHex4Digits(GetUi16(g + 4), s); s += 4; *s++ = '-'; - ConvertUInt16ToHex4Digits(GetUi16(g + 6), s); s += 4; *s++ = '-'; - for (unsigned i = 0; i < 8; i++) + if (size) { - if (i == 2) - *s++ = '-'; - ConvertByteToHex2Digits(g[8 + i], s); - s += 2; + const Byte *lim = src + size; + do + { + const unsigned b = *src++; + dest[0] = GET_HEX_CHAR_LOWER(b >> 4); + dest[1] = GET_HEX_CHAR_LOWER(b & 0xF); + dest += 2; + } + while (src != lim); } - *s = 0; - return s; + *dest = 0; } -char *RawLeGuidToString_Braced(const Byte *g, char *s) throw() +void ConvertDataToHex_Upper(char *dest, const Byte *src, size_t size) throw() { - *s++ = '{'; - s = RawLeGuidToString(g, s); - *s++ = '}'; - *s = 0; - return s; + if (size) + { + const Byte *lim = src + size; + do + { + const unsigned b = *src++; + dest[0] = GET_HEX_CHAR_UPPER(b >> 4); + dest[1] = GET_HEX_CHAR_UPPER(b & 0xF); + dest += 2; + } + while (src != lim); + } + *dest = 0; } diff --git a/CPP/Common/IntToString.h b/CPP/Common/IntToString.h index f4fc662..2f096d6 100644 --- a/CPP/Common/IntToString.h +++ b/CPP/Common/IntToString.h @@ -12,19 +12,43 @@ char * ConvertUInt64ToString(UInt64 value, char *s) throw(); wchar_t * ConvertUInt32ToString(UInt32 value, wchar_t *s) throw(); wchar_t * ConvertUInt64ToString(UInt64 value, wchar_t *s) throw(); +void ConvertInt64ToString(Int64 value, char *s) throw(); +void ConvertInt64ToString(Int64 value, wchar_t *s) throw(); void ConvertUInt64ToOct(UInt64 value, char *s) throw(); +extern const char k_Hex_Upper[16]; +extern const char k_Hex_Lower[16]; + +#define GET_HEX_CHAR_UPPER(t) (k_Hex_Upper[t]) +#define GET_HEX_CHAR_LOWER(t) (k_Hex_Lower[t]) +/* +// #define GET_HEX_CHAR_UPPER(t) ((char)(((t < 10) ? ('0' + t) : ('A' + (t - 10))))) +static inline unsigned GetHex_Lower(unsigned v) +{ + const unsigned v0 = v + '0'; + v += 'a' - 10; + if (v < 'a') + v = v0; + return v; +} +static inline char GetHex_Upper(unsigned v) +{ + return (char)((v < 10) ? ('0' + v) : ('A' + (v - 10))); +} +*/ + + void ConvertUInt32ToHex(UInt32 value, char *s) throw(); void ConvertUInt64ToHex(UInt64 value, char *s) throw(); void ConvertUInt32ToHex8Digits(UInt32 value, char *s) throw(); // void ConvertUInt32ToHex8Digits(UInt32 value, wchar_t *s) throw(); -void ConvertInt64ToString(Int64 value, char *s) throw(); -void ConvertInt64ToString(Int64 value, wchar_t *s) throw(); - // use RawLeGuid only for RAW bytes that contain stored GUID as Little-endian. char *RawLeGuidToString(const Byte *guid, char *s) throw(); char *RawLeGuidToString_Braced(const Byte *guid, char *s) throw(); +void ConvertDataToHex_Lower(char *dest, const Byte *src, size_t size) throw(); +void ConvertDataToHex_Upper(char *dest, const Byte *src, size_t size) throw(); + #endif diff --git a/CPP/Common/ListFileUtils.cpp b/CPP/Common/ListFileUtils.cpp index e43dbc9..714b275 100644 --- a/CPP/Common/ListFileUtils.cpp +++ b/CPP/Common/ListFileUtils.cpp @@ -76,26 +76,26 @@ bool ReadNamesFromListFile2(CFSTR fileName, UStringVector &strings, UINT codePag return false; file.Close(); - const unsigned num = (unsigned)fileSize / 2; - wchar_t *p = u.GetBuf(num); + const size_t num = (size_t)fileSize / 2; + wchar_t *p = u.GetBuf((unsigned)num); if (codePage == Z7_WIN_CP_UTF16) - for (unsigned i = 0; i < num; i++) + for (size_t i = 0; i < num; i++) { - wchar_t c = GetUi16(buf + (size_t)i * 2); + const wchar_t c = GetUi16(buf.ConstData() + (size_t)i * 2); if (c == 0) return false; p[i] = c; } else - for (unsigned i = 0; i < num; i++) + for (size_t i = 0; i < num; i++) { - wchar_t c = (wchar_t)GetBe16(buf + (size_t)i * 2); + const wchar_t c = (wchar_t)GetBe16(buf.ConstData() + (size_t)i * 2); if (c == 0) return false; p[i] = c; } p[num] = 0; - u.ReleaseBuf_SetLen(num); + u.ReleaseBuf_SetLen((unsigned)num); } else { diff --git a/CPP/Common/MyBuffer.h b/CPP/Common/MyBuffer.h index bc829f4..80f0205 100644 --- a/CPP/Common/MyBuffer.h +++ b/CPP/Common/MyBuffer.h @@ -51,6 +51,12 @@ template class CBuffer operator T *() { return _items; } operator const T *() const { return _items; } + const T* ConstData() const { return _items; } + T* NonConstData() const { return _items; } + T* NonConstData() { return _items; } + // const T* Data() const { return _items; } + // T* Data() { return _items; } + size_t Size() const { return _size; } void Alloc(size_t size) @@ -140,15 +146,15 @@ bool operator!=(const CBuffer& b1, const CBuffer& b2) // typedef CBuffer CCharBuffer; // typedef CBuffer CWCharBuffer; -typedef CBuffer CByteBuffer; +typedef CBuffer CByteBuffer; class CByteBuffer_Wipe: public CByteBuffer { Z7_CLASS_NO_COPY(CByteBuffer_Wipe) public: - // CByteBuffer_Wipe(): CBuffer() {} - CByteBuffer_Wipe(size_t size): CBuffer(size) {} + // CByteBuffer_Wipe(): CBuffer() {} + CByteBuffer_Wipe(size_t size): CBuffer(size) {} ~CByteBuffer_Wipe() { Wipe(); } }; @@ -181,6 +187,11 @@ template class CObjArray operator T *() { return _items; } operator const T *() const { return _items; } + const T* ConstData() const { return _items; } + T* NonConstData() const { return _items; } + T* NonConstData() { return _items; } + // const T* Data() const { return _items; } + // T* Data() { return _items; } void Alloc(size_t newSize) { @@ -191,7 +202,7 @@ template class CObjArray } }; -typedef CObjArray CByteArr; +typedef CObjArray CByteArr; typedef CObjArray CBoolArr; typedef CObjArray CIntArr; typedef CObjArray CUIntArr; diff --git a/CPP/Common/MyBuffer2.h b/CPP/Common/MyBuffer2.h index 23394f8..1ec8ffb 100644 --- a/CPP/Common/MyBuffer2.h +++ b/CPP/Common/MyBuffer2.h @@ -65,13 +65,13 @@ class CAlignedBuffer1 public: ~CAlignedBuffer1() { - ISzAlloc_Free(&g_AlignedAlloc, _data); + z7_AlignedFree(_data); } CAlignedBuffer1(size_t size) { _data = NULL; - _data = (Byte *)ISzAlloc_Alloc(&g_AlignedAlloc, size); + _data = (Byte *)z7_AlignedAlloc(size); if (!_data) throw 1; } @@ -92,21 +92,23 @@ class CAlignedBuffer CAlignedBuffer(): _data(NULL), _size(0) {} ~CAlignedBuffer() { - ISzAlloc_Free(&g_AlignedAlloc, _data); + z7_AlignedFree(_data); } + /* CAlignedBuffer(size_t size): _size(0) { _data = NULL; - _data = (Byte *)ISzAlloc_Alloc(&g_AlignedAlloc, size); + _data = (Byte *)z7_AlignedAlloc(size); if (!_data) throw 1; _size = size; } + */ void Free() { - ISzAlloc_Free(&g_AlignedAlloc, _data); + z7_AlignedFree(_data); _data = NULL; _size = 0; } @@ -120,10 +122,10 @@ class CAlignedBuffer { if (!_data || size != _size) { - ISzAlloc_Free(&g_AlignedAlloc, _data); + z7_AlignedFree(_data); _size = 0; _data = NULL; - _data = (Byte *)ISzAlloc_Alloc(&g_AlignedAlloc, size); + _data = (Byte *)z7_AlignedAlloc(size); if (_data) _size = size; } @@ -133,10 +135,29 @@ class CAlignedBuffer { if (!_data || size > _size) { - ISzAlloc_Free(&g_AlignedAlloc, _data); + z7_AlignedFree(_data); _size = 0; _data = NULL; - _data = (Byte *)ISzAlloc_Alloc(&g_AlignedAlloc, size); + _data = (Byte *)z7_AlignedAlloc(size); + if (_data) + _size = size; + } + } + + // (size <= size_max) + void AllocAtLeast_max(size_t size, size_t size_max) + { + if (!_data || size > _size) + { + z7_AlignedFree(_data); + _size = 0; + _data = NULL; + if (size_max < size) size_max = size; // optional check + const size_t delta = size / 2; + size += delta; + if (size < delta || size > size_max) + size = size_max; + _data = (Byte *)z7_AlignedAlloc(size); if (_data) _size = size; } diff --git a/CPP/Common/MyCom.h b/CPP/Common/MyCom.h index 65c4330..a3cc3c8 100644 --- a/CPP/Common/MyCom.h +++ b/CPP/Common/MyCom.h @@ -17,6 +17,7 @@ class CMyComPtr ~CMyComPtr() { if (_p) _p->Release(); } void Release() { if (_p) { _p->Release(); _p = NULL; } } operator T*() const { return (T*)_p; } + T* Interface() const { return (T*)_p; } // T& operator*() const { return *_p; } T** operator&() { return &_p; } T* operator->() const { return _p; } @@ -68,6 +69,112 @@ class CMyComPtr } }; + +template +class CMyComPtr2 +{ + cls* _p; + + CMyComPtr2(const CMyComPtr2& lp); + CMyComPtr2(cls* p); + CMyComPtr2(iface* p); + iface* operator=(const CMyComPtr2& lp); + iface* operator=(cls* p); + iface* operator=(iface* p); +public: + CMyComPtr2(): _p(NULL) {} + ~CMyComPtr2() + { + if (_p) + { + iface *ip = _p; + ip->Release(); + } + } + // void Release() { if (_p) { (iface *)_p->Release(); _p = NULL; } } + cls* operator->() const { return _p; } + cls* ClsPtr() const { return _p; } + operator iface*() const + { + iface *ip = _p; + return ip; + } + iface* Interface() const + { + iface *ip = _p; + return ip; + } + // operator bool() const { return _p != NULL; } + bool IsDefined() const { return _p != NULL; } + void Create_if_Empty() + { + if (!_p) + { + _p = new cls; + iface *ip = _p; + ip->AddRef(); + } + } + iface* Detach() + { + iface *ip = _p; + _p = NULL; + return ip; + } + void SetFromCls(cls *src) + { + if (src) + { + iface *ip = src; + ip->AddRef(); + } + if (_p) + { + iface *ip = _p; + ip->Release(); + } + _p = src; + } +}; + + +template +class CMyComPtr2_Create +{ + cls* _p; + + CMyComPtr2_Create(const CMyComPtr2_Create& lp); + CMyComPtr2_Create(cls* p); + CMyComPtr2_Create(iface* p); + iface* operator=(const CMyComPtr2_Create& lp); + iface* operator=(cls* p); + iface* operator=(iface* p); +public: + CMyComPtr2_Create(): _p(new cls) + { + iface *ip = _p; + ip->AddRef(); + } + ~CMyComPtr2_Create() + { + iface *ip = _p; + ip->Release(); + } + cls* operator->() const { return _p; } + cls* ClsPtr() const { return _p; } + operator iface*() const + { + iface *ip = _p; + return ip; + } + iface* Interface() const + { + iface *ip = _p; + return ip; + } +}; + + #define Z7_DECL_CMyComPtr_QI_FROM(i, v, unk) \ CMyComPtr v; (unk)->QueryInterface(IID_ ## i, (void **)&v); @@ -234,16 +341,56 @@ class CMyUnknownImp Z7_COM_QI_ENTRY_UNKNOWN(i) \ Z7_COM_QI_ENTRY(i) -#define Z7_COM_QI_END \ + +#define Z7_COM_ADDREF_RELEASE_MT \ + private: \ + STDMETHOD_(ULONG, AddRef)() Z7_override Z7_final \ + { return (ULONG)InterlockedIncrement((LONG *)&_m_RefCount); } \ + STDMETHOD_(ULONG, Release)() Z7_override Z7_final \ + { const LONG v = InterlockedDecrement((LONG *)&_m_RefCount); \ + if (v != 0) return (ULONG)v; \ + delete this; return 0; } + +#define Z7_COM_QI_END_MT \ else return E_NOINTERFACE; \ - ++_m_RefCount; /* AddRef(); */ return S_OK; } + InterlockedIncrement((LONG *)&_m_RefCount); /* AddRef(); */ return S_OK; } + +// you can define Z7_COM_USE_ATOMIC, +// if you want to call Release() from different threads (for example, for .NET code) +// #define Z7_COM_USE_ATOMIC + +#if defined(Z7_COM_USE_ATOMIC) && !defined(Z7_ST) + +#ifndef _WIN32 +#if 0 +#include "../../C/Threads.h" +#else +EXTERN_C_BEGIN +LONG InterlockedIncrement(LONG volatile *addend); +LONG InterlockedDecrement(LONG volatile *addend); +EXTERN_C_END +#endif +#endif // _WIN32 + +#define Z7_COM_ADDREF_RELEASE Z7_COM_ADDREF_RELEASE_MT +#define Z7_COM_QI_END Z7_COM_QI_END_MT + +#else // !Z7_COM_USE_ATOMIC #define Z7_COM_ADDREF_RELEASE \ private: \ STDMETHOD_(ULONG, AddRef)() throw() Z7_override Z7_final \ { return ++_m_RefCount; } \ STDMETHOD_(ULONG, Release)() throw() Z7_override Z7_final \ - { if (--_m_RefCount != 0) return _m_RefCount; delete this; return 0; } \ + { if (--_m_RefCount != 0) return _m_RefCount; \ + delete this; return 0; } + +#define Z7_COM_QI_END \ + else return E_NOINTERFACE; \ + ++_m_RefCount; /* AddRef(); */ return S_OK; } + +#endif // !Z7_COM_USE_ATOMIC + #define Z7_COM_UNKNOWN_IMP_SPEC(i) \ Z7_COM_QI_BEGIN \ diff --git a/CPP/Common/MyGuidDef.h b/CPP/Common/MyGuidDef.h index ab9993b..3aa5266 100644 --- a/CPP/Common/MyGuidDef.h +++ b/CPP/Common/MyGuidDef.h @@ -13,7 +13,7 @@ typedef struct { UInt32 Data1; UInt16 Data2; UInt16 Data3; - unsigned char Data4[8]; + Byte Data4[8]; } GUID; #ifdef __cplusplus @@ -32,7 +32,7 @@ typedef GUID CLSID; inline int operator==(REFGUID g1, REFGUID g2) { for (unsigned i = 0; i < sizeof(g1); i++) - if (((const unsigned char *)&g1)[i] != ((const unsigned char *)&g2)[i]) + if (((const Byte *)&g1)[i] != ((const Byte *)&g2)[i]) return 0; return 1; } diff --git a/CPP/Common/MyMap.cpp b/CPP/Common/MyMap.cpp index 0a200f0..d130aec 100644 --- a/CPP/Common/MyMap.cpp +++ b/CPP/Common/MyMap.cpp @@ -103,7 +103,7 @@ bool CMap32::Set(UInt32 key, UInt32 value) n.IsLeaf[newBit] = 1; n.IsLeaf[1 - newBit] = 0; n.Keys[newBit] = key; - n.Keys[1 - newBit] = Nodes.Size(); + n.Keys[1 - newBit] = (UInt32)Nodes.Size(); Nodes.Add(e2); return false; } @@ -130,7 +130,7 @@ bool CMap32::Set(UInt32 key, UInt32 value) e2.Len = (UInt16)(bitPos - (1 + i)); n.IsLeaf[bit] = 0; - n.Keys[bit] = Nodes.Size(); + n.Keys[bit] = (UInt32)Nodes.Size(); Nodes.Add(e2); return false; diff --git a/CPP/Common/MyString.cpp b/CPP/Common/MyString.cpp index 51c1c3b..b5f7e52 100644 --- a/CPP/Common/MyString.cpp +++ b/CPP/Common/MyString.cpp @@ -62,7 +62,7 @@ void MyStringUpper_Ascii(char *s) throw() { for (;;) { - char c = *s; + const char c = *s; if (c == 0) return; *s++ = MyCharUpper_Ascii(c); @@ -73,7 +73,7 @@ void MyStringUpper_Ascii(wchar_t *s) throw() { for (;;) { - wchar_t c = *s; + const wchar_t c = *s; if (c == 0) return; *s++ = MyCharUpper_Ascii(c); @@ -85,7 +85,7 @@ void MyStringLower_Ascii(char *s) throw() { for (;;) { - char c = *s; + const char c = *s; if (c == 0) return; *s++ = MyCharLower_Ascii(c); @@ -96,7 +96,7 @@ void MyStringLower_Ascii(wchar_t *s) throw() { for (;;) { - wchar_t c = *s; + const wchar_t c = *s; if (c == 0) return; *s++ = MyCharLower_Ascii(c); @@ -190,8 +190,8 @@ bool IsString1PrefixedByString2(const char *s1, const char *s2) throw() { for (;;) { - const unsigned char c2 = (unsigned char)*s2++; if (c2 == 0) return true; - const unsigned char c1 = (unsigned char)*s1++; if (c1 != c2) return false; + const char c2 = *s2++; if (c2 == 0) return true; + const char c1 = *s1++; if (c1 != c2) return false; } } @@ -402,6 +402,7 @@ void AString::InsertSpace(unsigned &index, unsigned size) } #define k_Alloc_Len_Limit (0x40000000 - 2) +// #define k_Alloc_Len_Limit (((unsigned)1 << (sizeof(unsigned) * 8 - 2)) - 2) void AString::ReAlloc(unsigned newLimit) { @@ -413,9 +414,14 @@ void AString::ReAlloc(unsigned newLimit) _limit = newLimit; } +#define THROW_STRING_ALLOC_EXCEPTION { throw 20130220; } + +#define CHECK_STRING_ALLOC_LEN(len) \ + { if ((len) > k_Alloc_Len_Limit) THROW_STRING_ALLOC_EXCEPTION } + void AString::ReAlloc2(unsigned newLimit) { - if (newLimit > k_Alloc_Len_Limit) throw 20130220; + CHECK_STRING_ALLOC_LEN(newLimit) // MY_STRING_REALLOC(_chars, char, (size_t)newLimit + 1, 0); char *newBuf = MY_STRING_NEW_char((size_t)newLimit + 1); newBuf[0] = 0; @@ -433,6 +439,7 @@ void AString::SetStartLen(unsigned len) _limit = len; } +Z7_NO_INLINE void AString::Grow_1() { unsigned next = _len; @@ -443,7 +450,7 @@ void AString::Grow_1() if (next < _len || next > k_Alloc_Len_Limit) next = k_Alloc_Len_Limit; if (next <= _len) - throw 20130220; + THROW_STRING_ALLOC_EXCEPTION ReAlloc(next); // Grow(1); } @@ -461,7 +468,7 @@ void AString::Grow(unsigned n) if (next < _len || next > k_Alloc_Len_Limit) next = k_Alloc_Len_Limit; if (next <= _len || next - _len < n) - throw 20130220; + THROW_STRING_ALLOC_EXCEPTION ReAlloc(next); } @@ -638,12 +645,14 @@ void AString::SetFromBstr_if_Ascii(BSTR s) } */ +void AString::Add_Char(char c) { operator+=(c); } void AString::Add_Space() { operator+=(' '); } void AString::Add_Space_if_NotEmpty() { if (!IsEmpty()) Add_Space(); } void AString::Add_LF() { operator+=('\n'); } void AString::Add_Slash() { operator+=('/'); } void AString::Add_Dot() { operator+=('.'); } void AString::Add_Minus() { operator+=('-'); } +void AString::Add_Colon() { operator+=(':'); } AString &AString::operator+=(const char *s) { @@ -696,6 +705,7 @@ void AString::SetFrom(const char *s, unsigned len) // no check { if (len > _limit) { + CHECK_STRING_ALLOC_LEN(len) char *newBuf = MY_STRING_NEW_char((size_t)len + 1); MY_STRING_DELETE(_chars) _chars = newBuf; @@ -707,6 +717,12 @@ void AString::SetFrom(const char *s, unsigned len) // no check _len = len; } +void AString::SetFrom_Chars_SizeT(const char *s, size_t len) +{ + CHECK_STRING_ALLOC_LEN(len) + SetFrom(s, (unsigned)len); +} + void AString::SetFrom_CalcLen(const char *s, unsigned len) // no check { unsigned i; @@ -906,8 +922,8 @@ void AString::Replace(const AString &oldString, const AString &newString) return; // 0; if (oldString == newString) return; // 0; - unsigned oldLen = oldString.Len(); - unsigned newLen = newString.Len(); + const unsigned oldLen = oldString.Len(); + const unsigned newLen = newString.Len(); // unsigned number = 0; int pos = 0; while ((unsigned)pos < _len) @@ -1011,7 +1027,7 @@ void UString::ReAlloc(unsigned newLimit) void UString::ReAlloc2(unsigned newLimit) { - if (newLimit > k_Alloc_Len_Limit) throw 20130221; + CHECK_STRING_ALLOC_LEN(newLimit) // MY_STRING_REALLOC(_chars, wchar_t, newLimit + 1, 0); wchar_t *newBuf = MY_STRING_NEW_wchar_t((size_t)newLimit + 1); newBuf[0] = 0; @@ -1029,6 +1045,7 @@ void UString::SetStartLen(unsigned len) _limit = len; } +Z7_NO_INLINE void UString::Grow_1() { unsigned next = _len; @@ -1039,7 +1056,7 @@ void UString::Grow_1() if (next < _len || next > k_Alloc_Len_Limit) next = k_Alloc_Len_Limit; if (next <= _len) - throw 20130220; + THROW_STRING_ALLOC_EXCEPTION ReAlloc(next); } @@ -1056,7 +1073,7 @@ void UString::Grow(unsigned n) if (next < _len || next > k_Alloc_Len_Limit) next = k_Alloc_Len_Limit; if (next <= _len || next - _len < n) - throw 20130220; + THROW_STRING_ALLOC_EXCEPTION ReAlloc(next - 1); } @@ -1214,6 +1231,7 @@ void UString::SetFrom(const wchar_t *s, unsigned len) // no check { if (len > _limit) { + CHECK_STRING_ALLOC_LEN(len) wchar_t *newBuf = MY_STRING_NEW_wchar_t((size_t)len + 1); MY_STRING_DELETE(_chars) _chars = newBuf; @@ -1238,7 +1256,7 @@ void UString::SetFromBstr(LPCOLESTR s) if (c >= 0xd800 && c < 0xdc00 && i + 1 != len) { wchar_t c2 = s[i]; - if (c2 >= 0xdc00 && c2 < 0x10000) + if (c2 >= 0xdc00 && c2 < 0xe000) { c = 0x10000 + ((c & 0x3ff) << 10) + (c2 & 0x3ff); i++; @@ -1269,7 +1287,7 @@ void UString::SetFromBstr(LPCOLESTR s) if (c >= 0xd800 && c < 0xdc00 && i + 1 != len) { wchar_t c2 = *s; - if (c2 >= 0xdc00 && c2 < 0x10000) + if (c2 >= 0xdc00 && c2 < 0xe000) { s++; c = 0x10000 + ((c & 0x3ff) << 10) + (c2 & 0x3ff); @@ -1305,21 +1323,15 @@ UString &UString::operator=(const char *s) return *this; } +void UString::Add_Char(char c) { operator+=((wchar_t)(unsigned char)c); } +// void UString::Add_WChar(wchar_t c) { operator+=(c); } void UString::Add_Dot() { operator+=(L'.'); } void UString::Add_Space() { operator+=(L' '); } +void UString::Add_Minus() { operator+=(L'-'); } +void UString::Add_Colon() { operator+=(L':'); } +void UString::Add_LF() { operator+=(L'\n'); } void UString::Add_Space_if_NotEmpty() { if (!IsEmpty()) Add_Space(); } -void UString::Add_LF() -{ - if (_limit == _len) - Grow_1(); - unsigned len = _len; - wchar_t *chars = _chars; - chars[len++] = L'\n'; - chars[len] = 0; - _len = len; -} - UString &UString::operator+=(const wchar_t *s) { unsigned len = MyStringLen(s); @@ -1597,7 +1609,7 @@ void UString::DeleteFrontal(unsigned num) throw() void UString2::ReAlloc2(unsigned newLimit) { // wrong (_len) is allowed after this function - if (newLimit > k_Alloc_Len_Limit) throw 20130221; + CHECK_STRING_ALLOC_LEN(newLimit) // MY_STRING_REALLOC(_chars, wchar_t, newLimit + 1, 0); if (_chars) { @@ -1805,7 +1817,7 @@ bool CStringFinder::FindWord_In_LowCaseAsciiList_NoCase(const char *p, const wch break; if (c <= 0x20 || c > 0x7f) return false; - _temp += (char)MyCharLower_Ascii((char)c); + _temp.Add_Char((char)MyCharLower_Ascii((char)c)); } while (*p != 0) diff --git a/CPP/Common/MyString.h b/CPP/Common/MyString.h index e5ce18a..ba9914e 100644 --- a/CPP/Common/MyString.h +++ b/CPP/Common/MyString.h @@ -281,6 +281,8 @@ class AString void ReAlloc(unsigned newLimit); void ReAlloc2(unsigned newLimit); void SetStartLen(unsigned len); + + Z7_NO_INLINE void Grow_1(); void Grow(unsigned n); @@ -373,6 +375,8 @@ class AString void SetFromWStr_if_Ascii(const wchar_t *s); // void SetFromBstr_if_Ascii(BSTR s); +// private: + Z7_FORCE_INLINE AString &operator+=(char c) { if (_limit == _len) @@ -384,14 +388,16 @@ class AString _len = len; return *this; } - +public: void Add_Space(); void Add_Space_if_NotEmpty(); void Add_OptSpaced(const char *s); + void Add_Char(char c); void Add_LF(); void Add_Slash(); void Add_Dot(); void Add_Minus(); + void Add_Colon(); void Add_PathSepar() { operator+=(CHAR_PATH_SEPARATOR); } AString &operator+=(const char *s); @@ -402,6 +408,7 @@ class AString void AddFrom(const char *s, unsigned len); // no check void SetFrom(const char *s, unsigned len); // no check + void SetFrom_Chars_SizeT(const char* s, size_t len); // no check void SetFrom(const char* s, int len) // no check { SetFrom(s, (unsigned)len); // no check @@ -668,6 +675,8 @@ class UString UString &operator=(const char *s); UString &operator=(const AString &s) { return operator=(s.Ptr()); } +// private: + Z7_FORCE_INLINE UString &operator+=(wchar_t c) { if (_limit == _len) @@ -680,12 +689,17 @@ class UString return *this; } - UString &operator+=(char c) { return (*this)+=((wchar_t)(unsigned char)c); } - +private: + UString &operator+=(char c); // { return (*this)+=((wchar_t)(unsigned char)c); } +public: + void Add_Char(char c); + // void Add_WChar(wchar_t c); void Add_Space(); void Add_Space_if_NotEmpty(); void Add_LF(); void Add_Dot(); + void Add_Minus(); + void Add_Colon(); void Add_PathSepar() { operator+=(WCHAR_PATH_SEPARATOR); } UString &operator+=(const wchar_t *s); diff --git a/CPP/Common/MyTypes.h b/CPP/Common/MyTypes.h index 8f44f67..eadc9a4 100644 --- a/CPP/Common/MyTypes.h +++ b/CPP/Common/MyTypes.h @@ -3,10 +3,11 @@ #ifndef ZIP7_INC_COMMON_MY_TYPES_H #define ZIP7_INC_COMMON_MY_TYPES_H +#include "Common0.h" #include "../../C/7zTypes.h" -#include "Common.h" -typedef int HRes; +// typedef int HRes; +// typedef HRESULT HRes; struct CBoolPair { diff --git a/CPP/Common/MyVector.h b/CPP/Common/MyVector.h index 9ee7105..a772785 100644 --- a/CPP/Common/MyVector.h +++ b/CPP/Common/MyVector.h @@ -258,7 +258,7 @@ class CRecordVector { if (index != 0) { - T temp = _items[index]; + const T temp = _items[index]; memmove(_items + 1, _items, (size_t)index * sizeof(T)); _items[0] = temp; } @@ -268,15 +268,29 @@ class CRecordVector T& operator[](unsigned index) { return _items[index]; } const T& operator[](int index) const { return _items[(unsigned)index]; } T& operator[](int index) { return _items[(unsigned)index]; } + + const T* ConstData() const { return _items; } + T* NonConstData() const { return _items; } + T* NonConstData() { return _items; } + + const T* Data() const { return _items; } + T* Data() { return _items; } + + const T& FrontItem() const { return _items[0]; } + T& FrontItem() { return _items[0]; } + /* + const T Front() const { return _items[0]; } + T Front() { return _items[0]; } const T& Front() const { return _items[0]; } T& Front() { return _items[0]; } + */ const T& Back() const { return _items[(size_t)_size - 1]; } T& Back() { return _items[(size_t)_size - 1]; } /* void Swap(unsigned i, unsigned j) { - T temp = _items[i]; + const T temp = _items[i]; _items[i] = _items[j]; _items[j] = temp; } @@ -368,7 +382,7 @@ class CRecordVector static void SortRefDown(T* p, unsigned k, unsigned size, int (*compare)(const T*, const T*, void *), void *param) { - T temp = p[k]; + const T temp = p[k]; for (;;) { unsigned s = (k << 1); @@ -389,16 +403,16 @@ class CRecordVector unsigned size = _size; if (size <= 1) return; - T* p = (&Front()) - 1; + T* p = _items - 1; { unsigned i = size >> 1; do SortRefDown(p, i, size, compare, param); - while (--i != 0); + while (--i); } do { - T temp = p[size]; + const T temp = p[size]; p[size--] = p[1]; p[1] = temp; SortRefDown(p, 1, size, compare, param); @@ -408,7 +422,7 @@ class CRecordVector static void SortRefDown2(T* p, unsigned k, unsigned size) { - T temp = p[k]; + const T temp = p[k]; for (;;) { unsigned s = (k << 1); @@ -429,16 +443,16 @@ class CRecordVector unsigned size = _size; if (size <= 1) return; - T* p = (&Front()) - 1; + T* p = _items - 1; { unsigned i = size >> 1; do SortRefDown2(p, i, size); - while (--i != 0); + while (--i); } do { - T temp = p[size]; + const T temp = p[size]; p[size--] = p[1]; p[1] = temp; SortRefDown2(p, 1, size); diff --git a/CPP/Common/MyWindows.cpp b/CPP/Common/MyWindows.cpp index ae284eb..a1dfbef 100644 --- a/CPP/Common/MyWindows.cpp +++ b/CPP/Common/MyWindows.cpp @@ -88,21 +88,21 @@ BSTR SysAllocString(const OLECHAR *s) void SysFreeString(BSTR bstr) { if (bstr) - FreeForBSTR((CBstrSizeType *)bstr - 1); + FreeForBSTR((CBstrSizeType *)(void *)bstr - 1); } UINT SysStringByteLen(BSTR bstr) { if (!bstr) return 0; - return *((CBstrSizeType *)bstr - 1); + return *((CBstrSizeType *)(void *)bstr - 1); } UINT SysStringLen(BSTR bstr) { if (!bstr) return 0; - return *((CBstrSizeType *)bstr - 1) / (UINT)sizeof(OLECHAR); + return *((CBstrSizeType *)(void *)bstr - 1) / (UINT)sizeof(OLECHAR); } diff --git a/CPP/Common/MyWindows.h b/CPP/Common/MyWindows.h index a76e14b..da5370b 100644 --- a/CPP/Common/MyWindows.h +++ b/CPP/Common/MyWindows.h @@ -66,8 +66,8 @@ typedef int BOOL; typedef Int64 LONGLONG; typedef UInt64 ULONGLONG; -typedef struct _LARGE_INTEGER { LONGLONG QuadPart; } LARGE_INTEGER; -typedef struct _ULARGE_INTEGER { ULONGLONG QuadPart; } ULARGE_INTEGER; +typedef struct { LONGLONG QuadPart; } LARGE_INTEGER; +typedef struct { ULONGLONG QuadPart; } ULARGE_INTEGER; typedef const CHAR *LPCSTR; typedef CHAR TCHAR; @@ -79,7 +79,7 @@ typedef OLECHAR *BSTR; typedef const OLECHAR *LPCOLESTR; typedef OLECHAR *LPOLESTR; -typedef struct _FILETIME +typedef struct { DWORD dwLowDateTime; DWORD dwHighDateTime; @@ -292,7 +292,7 @@ typedef enum tagSTREAM_SEEK -typedef struct _SYSTEMTIME +typedef struct { WORD wYear; WORD wMonth; @@ -312,12 +312,13 @@ BOOL WINAPI FileTimeToSystemTime(const FILETIME *fileTime, SYSTEMTIME *systemTim DWORD GetTickCount(); +/* #define CREATE_NEW 1 #define CREATE_ALWAYS 2 #define OPEN_EXISTING 3 #define OPEN_ALWAYS 4 #define TRUNCATE_EXISTING 5 - +*/ #endif // _WIN32 diff --git a/CPP/Common/MyXml.cpp b/CPP/Common/MyXml.cpp index a879d34..cc891fc 100644 --- a/CPP/Common/MyXml.cpp +++ b/CPP/Common/MyXml.cpp @@ -3,6 +3,7 @@ #include "StdAfx.h" #include "MyXml.h" +#include "StringToInt.h" static bool IsValidChar(char c) { @@ -30,7 +31,7 @@ int CXmlItem::FindProp(const char *propName) const throw() AString CXmlItem::GetPropVal(const char *propName) const { - int index = FindProp(propName); + const int index = FindProp(propName); if (index >= 0) return Props[(unsigned)index].Value; return AString(); @@ -49,6 +50,17 @@ int CXmlItem::FindSubTag(const char *tag) const throw() return -1; } +const CXmlItem *CXmlItem::FindSubTag_GetPtr(const char *tag) const throw() +{ + FOR_VECTOR (i, SubItems) + { + const CXmlItem *p = &SubItems[i]; + if (p->IsTagged(tag)) + return p; + } + return NULL; +} + AString CXmlItem::GetSubString() const { if (SubItems.Size() == 1) @@ -73,9 +85,9 @@ const AString * CXmlItem::GetSubStringPtr() const throw() AString CXmlItem::GetSubStringForTag(const char *tag) const { - int index = FindSubTag(tag); - if (index >= 0) - return SubItems[(unsigned)index].GetSubString(); + const CXmlItem *item = FindSubTag_GetPtr(tag); + if (item) + return item->GetSubString(); return AString(); } @@ -92,11 +104,14 @@ const char * CXmlItem::ParseItem(const char *s, int numAllowedLevels) } if (*s == 0) return NULL; - if (s != beg) { - IsTag = false; - Name.SetFrom(beg, (unsigned)(s - beg)); - return s; + const size_t num = (size_t)(s - beg); + if (num) + { + IsTag = false; + Name.SetFrom_Chars_SizeT(beg, num); + return s; + } } IsTag = true; @@ -110,7 +125,7 @@ const char * CXmlItem::ParseItem(const char *s, int numAllowedLevels) break; if (s == beg || *s == 0) return NULL; - Name.SetFrom(beg, (unsigned)(s - beg)); + Name.SetFrom_Chars_SizeT(beg, (size_t)(s - beg)); for (;;) { @@ -142,11 +157,12 @@ const char * CXmlItem::ParseItem(const char *s, int numAllowedLevels) } s += 2; - unsigned len = Name.Len(); + const unsigned len = Name.Len(); + const char *name = Name.Ptr(); for (unsigned i = 0; i < len; i++) - if (s[i] != Name[i]) + if (*s++ != *name++) return NULL; - s += len; + // s += len; if (s[0] != '>') return NULL; return s + 1; @@ -166,7 +182,7 @@ const char * CXmlItem::ParseItem(const char *s, int numAllowedLevels) } if (s == beg) return NULL; - prop.Name.SetFrom(beg, (unsigned)(s - beg)); + prop.Name.SetFrom_Chars_SizeT(beg, (size_t)(s - beg)); SKIP_SPACES(s) if (*s != '=') @@ -187,7 +203,7 @@ const char * CXmlItem::ParseItem(const char *s, int numAllowedLevels) break; s++; } - prop.Value.SetFrom(beg, (unsigned)(s - beg)); + prop.Value.SetFrom_Chars_SizeT(beg, (size_t)(s - beg)); s++; } } @@ -258,3 +274,76 @@ void CXml::AppendTo(AString &s) const Root.AppendTo(s); } */ + + +void z7_xml_DecodeString(AString &temp) +{ + char * const beg = temp.GetBuf(); + char *dest = beg; + const char *p = beg; + for (;;) + { + char c = *p++; + if (c == 0) + break; + if (c == '&') + { + if (p[0] == '#') + { + const char *end; + const UInt32 number = ConvertStringToUInt32(p + 1, &end); + if (*end == ';' && number != 0 && number <= 127) + { + p = end + 1; + c = (char)number; + } + } + else if ( + p[0] == 'a' && + p[1] == 'm' && + p[2] == 'p' && + p[3] == ';') + { + p += 4; + } + else if ( + p[0] == 'l' && + p[1] == 't' && + p[2] == ';') + { + p += 3; + c = '<'; + } + else if ( + p[0] == 'g' && + p[1] == 't' && + p[2] == ';') + { + p += 3; + c = '>'; + } + else if ( + p[0] == 'a' && + p[1] == 'p' && + p[2] == 'o' && + p[3] == 's' && + p[4] == ';') + { + p += 5; + c = '\''; + } + else if ( + p[0] == 'q' && + p[1] == 'u' && + p[2] == 'o' && + p[3] == 't' && + p[4] == ';') + { + p += 5; + c = '\"'; + } + } + *dest++ = c; + } + temp.ReleaseBuf_SetEnd((unsigned)(dest - beg)); +} diff --git a/CPP/Common/MyXml.h b/CPP/Common/MyXml.h index 5362602..b22d7e4 100644 --- a/CPP/Common/MyXml.h +++ b/CPP/Common/MyXml.h @@ -27,8 +27,8 @@ class CXmlItem AString GetSubString() const; const AString * GetSubStringPtr() const throw(); int FindSubTag(const char *tag) const throw(); + const CXmlItem *FindSubTag_GetPtr(const char *tag) const throw(); AString GetSubStringForTag(const char *tag) const; - void AppendTo(AString &s) const; }; @@ -40,4 +40,6 @@ struct CXml // void AppendTo(AString &s) const; }; +void z7_xml_DecodeString(AString &s); + #endif diff --git a/CPP/Common/NewHandler.cpp b/CPP/Common/NewHandler.cpp index c95833e..173b8eb 100644 --- a/CPP/Common/NewHandler.cpp +++ b/CPP/Common/NewHandler.cpp @@ -65,6 +65,13 @@ operator new(size_t size) return p; } + +#if defined(_MSC_VER) && _MSC_VER == 1600 +// vs2010 has no throw() by default ? +#pragma warning(push) +#pragma warning(disable : 4986) // 'operator delete': exception specification does not match previous declaration +#endif + void #ifdef _MSC_VER __cdecl @@ -76,6 +83,27 @@ operator delete(void *p) throw() ::free(p); } +/* we define operator delete(void *p, size_t n) because + vs2022 compiler uses delete(void *p, size_t n), and + we want to mix files from different compilers: + - old vc6 linker + - old vc6 complier + - new vs2022 complier +*/ +void +#ifdef _MSC_VER +__cdecl +#endif +operator delete(void *p, size_t n) throw() +{ + UNUSED_VAR(n) + ::free(p); +} + +#if defined(_MSC_VER) && _MSC_VER == 1600 +#pragma warning(pop) +#endif + /* void * #ifdef _MSC_VER @@ -205,11 +233,9 @@ operator delete(void *p) throw() a[i] = 0; */ HeapFree(GetProcessHeap(), 0, p); - if (numAllocs == 0) - numAllocs = numAllocs; // ERROR + // if (numAllocs == 0) numAllocs = numAllocs; // ERROR numAllocs--; - if (numAllocs == 0) - numAllocs = numAllocs; // OK: all objects were deleted + // if (numAllocs == 0) numAllocs = numAllocs; // OK: all objects were deleted printf("Free %d\n", numAllocs); LeaveCriticalSection(&cs); #else @@ -219,6 +245,22 @@ operator delete(void *p) throw() #endif } +void +#ifdef _MSC_VER +__cdecl +#endif +operator delete(void *p, size_t n) throw(); +void +#ifdef _MSC_VER +__cdecl +#endif +operator delete(void *p, size_t n) throw() +{ + UNUSED_VAR(n) + printf("delete_WITH_SIZE=%u, ptr = %p\n", (unsigned)n, p); + operator delete(p); +} + /* void * #ifdef _MSC_VER diff --git a/CPP/Common/NewHandler.h b/CPP/Common/NewHandler.h index 50f6d0a..5ba64b7 100644 --- a/CPP/Common/NewHandler.h +++ b/CPP/Common/NewHandler.h @@ -65,12 +65,35 @@ __cdecl #endif operator new(size_t size); +/* +#if 0 && defined(_MSC_VER) && _MSC_VER == 1600 + #define Z7_OPERATOR_DELETE_SPEC_THROW0 +#else + #define Z7_OPERATOR_DELETE_SPEC_THROW0 throw() +#endif +*/ +#if defined(_MSC_VER) && _MSC_VER == 1600 +#pragma warning(push) +#pragma warning(disable : 4986) // 'operator delete': exception specification does not match previous declaration +#endif + void #ifdef _MSC_VER __cdecl #endif operator delete(void *p) throw(); +void +#ifdef _MSC_VER +__cdecl +#endif +operator delete(void *p, size_t n) throw(); + +#if defined(_MSC_VER) && _MSC_VER == 1600 +#pragma warning(pop) +#endif + + #else #include diff --git a/CPP/Common/Random.cpp b/CPP/Common/Random.cpp index fbdb8c9..6ef9f1a 100644 --- a/CPP/Common/Random.cpp +++ b/CPP/Common/Random.cpp @@ -12,11 +12,11 @@ #include "Random.h" -void CRandom::Init(unsigned int seed) { srand(seed); } +void CRandom::Init(unsigned seed) { srand(seed); } void CRandom::Init() { - Init((unsigned int) + Init((unsigned) #ifdef _WIN32 GetTickCount() #else diff --git a/CPP/Common/Random.h b/CPP/Common/Random.h index 283869e..3fbb416 100644 --- a/CPP/Common/Random.h +++ b/CPP/Common/Random.h @@ -7,7 +7,7 @@ class CRandom { public: void Init(); - void Init(unsigned int seed); + void Init(unsigned seed); int Generate() const; }; diff --git a/CPP/Common/StdInStream.cpp b/CPP/Common/StdInStream.cpp index 7b209f1..944be34 100644 --- a/CPP/Common/StdInStream.cpp +++ b/CPP/Common/StdInStream.cpp @@ -48,15 +48,15 @@ bool CStdInStream::ScanAStringUntilNewLine(AString &s) s.Empty(); for (;;) { - int intChar = GetChar(); + const int intChar = GetChar(); if (intChar == EOF) return true; - char c = (char)intChar; + const char c = (char)intChar; if (c == 0) return false; if (c == '\n') return true; - s += c; + s.Add_Char(c); } } @@ -64,14 +64,14 @@ bool CStdInStream::ScanUStringUntilNewLine(UString &dest) { dest.Empty(); AString s; - bool res = ScanAStringUntilNewLine(s); + const bool res = ScanAStringUntilNewLine(s); int codePage = CodePage; if (codePage == -1) codePage = CP_OEMCP; - if (codePage == CP_UTF8) + if ((unsigned)codePage == CP_UTF8) ConvertUTF8ToUnicode(s, dest); else - MultiByteToUnicodeString2(dest, s, (UINT)codePage); + MultiByteToUnicodeString2(dest, s, (UINT)(unsigned)codePage); return res; } diff --git a/CPP/Common/StdOutStream.cpp b/CPP/Common/StdOutStream.cpp index cfa5fde..ba179d9 100644 --- a/CPP/Common/StdOutStream.cpp +++ b/CPP/Common/StdOutStream.cpp @@ -66,70 +66,158 @@ void CStdOutStream::Convert_UString_to_AString(const UString &src, AString &dest int codePage = CodePage; if (codePage == -1) codePage = CP_OEMCP; - if (codePage == CP_UTF8) + if ((unsigned)codePage == CP_UTF8) ConvertUnicodeToUTF8(src, dest); else - UnicodeStringToMultiByte2(dest, src, (UINT)codePage); + UnicodeStringToMultiByte2(dest, src, (UINT)(unsigned)codePage); } static const wchar_t kReplaceChar = '_'; +/* void CStdOutStream::Normalize_UString_LF_Allowed(UString &s) { - unsigned len = s.Len(); + if (!IsTerminalMode) + return; + + const unsigned len = s.Len(); wchar_t *d = s.GetBuf(); - if (IsTerminalMode) for (unsigned i = 0; i < len; i++) { - wchar_t c = d[i]; - if (c <= 13 && c >= 7 && c != '\n') + const wchar_t c = d[i]; + if (c == 0x1b || (c <= 13 && c >= 7 && c != '\n')) d[i] = kReplaceChar; } } +*/ void CStdOutStream::Normalize_UString(UString &s) { - unsigned len = s.Len(); + const unsigned len = s.Len(); wchar_t *d = s.GetBuf(); if (IsTerminalMode) for (unsigned i = 0; i < len; i++) { - wchar_t c = d[i]; - if (c <= 13 && c >= 7) + const wchar_t c = d[i]; + if ((c <= 13 && c >= 7) || c == 0x1b) d[i] = kReplaceChar; } else for (unsigned i = 0; i < len; i++) { - wchar_t c = d[i]; + const wchar_t c = d[i]; if (c == '\n') d[i] = kReplaceChar; } } -void CStdOutStream::NormalizePrint_UString(const UString &s, UString &tempU, AString &tempA) +void CStdOutStream::Normalize_UString_Path(UString &s) +{ + if (ListPathSeparatorSlash.Def) + { +#ifdef _WIN32 + if (ListPathSeparatorSlash.Val) + s.Replace(L'\\', L'/'); +#else + if (!ListPathSeparatorSlash.Val) + s.Replace(L'/', L'\\'); +#endif + } + Normalize_UString(s); +} + + +/* +void CStdOutStream::Normalize_UString(UString &src) +{ + const wchar_t *s = src.Ptr(); + const unsigned len = src.Len(); + unsigned i; + for (i = 0; i < len; i++) + { + const wchar_t c = s[i]; +#if 0 && !defined(_WIN32) + if (c == '\\') // IsTerminalMode && + break; +#endif + if ((unsigned)c < 0x20) + break; + } + if (i == len) + return; + + UString temp; + for (i = 0; i < len; i++) + { + wchar_t c = s[i]; +#if 0 && !defined(_WIN32) + if (c == '\\') + temp += (wchar_t)L'\\'; + else +#endif + if ((unsigned)c < 0x20) + { + if (c == '\n' + || (IsTerminalMode && (c == 0x1b || (c <= 13 && c >= 7)))) + { +#if 1 || defined(_WIN32) + c = (wchar_t)kReplaceChar; +#else + temp += (wchar_t)L'\\'; + if (c == '\n') c = L'n'; + else if (c == '\r') c = L'r'; + else if (c == '\a') c = L'a'; + else if (c == '\b') c = L'b'; + else if (c == '\t') c = L't'; + else if (c == '\v') c = L'v'; + else if (c == '\f') c = L'f'; + else + { + temp += (wchar_t)(L'0' + (unsigned)c / 64); + temp += (wchar_t)(L'0' + (unsigned)c / 8 % 8); + c = (wchar_t)(L'0' + (unsigned)c % 8); + } +#endif + } + } + temp += c; + } + src = temp; +} +*/ + +void CStdOutStream::NormalizePrint_UString_Path(const UString &s, UString &tempU, AString &tempA) { tempU = s; - Normalize_UString(tempU); + Normalize_UString_Path(tempU); PrintUString(tempU, tempA); } -void CStdOutStream::NormalizePrint_UString(const UString &s) +void CStdOutStream::NormalizePrint_UString_Path(const UString &s) { - NormalizePrint_wstr(s); + UString tempU; + AString tempA; + NormalizePrint_UString_Path(s, tempU, tempA); } -void CStdOutStream::NormalizePrint_wstr(const wchar_t *s) +void CStdOutStream::NormalizePrint_wstr_Path(const wchar_t *s) { UString tempU = s; - Normalize_UString(tempU); + Normalize_UString_Path(tempU); AString tempA; PrintUString(tempU, tempA); } +void CStdOutStream::NormalizePrint_UString(const UString &s) +{ + UString tempU = s; + Normalize_UString(tempU); + AString tempA; + PrintUString(tempU, tempA); +} CStdOutStream & CStdOutStream::operator<<(Int32 number) throw() { diff --git a/CPP/Common/StdOutStream.h b/CPP/Common/StdOutStream.h index bd15d7c..846b0db 100644 --- a/CPP/Common/StdOutStream.h +++ b/CPP/Common/StdOutStream.h @@ -14,6 +14,7 @@ class CStdOutStream // bool _streamIsOpen; public: bool IsTerminalMode; + CBoolPair ListPathSeparatorSlash; int CodePage; CStdOutStream(FILE *stream = NULL): @@ -21,7 +22,14 @@ class CStdOutStream // _streamIsOpen(false), IsTerminalMode(false), CodePage(-1) - {} + { + ListPathSeparatorSlash.Val = +#ifdef _WIN32 + false; +#else + true; +#endif + } // ~CStdOutStream() { Close(); } @@ -62,12 +70,13 @@ class CStdOutStream void PrintUString(const UString &s, AString &temp); void Convert_UString_to_AString(const UString &src, AString &dest); - void Normalize_UString_LF_Allowed(UString &s); void Normalize_UString(UString &s); + void Normalize_UString_Path(UString &s); - void NormalizePrint_UString(const UString &s, UString &tempU, AString &tempA); + void NormalizePrint_UString_Path(const UString &s, UString &tempU, AString &tempA); + void NormalizePrint_UString_Path(const UString &s); void NormalizePrint_UString(const UString &s); - void NormalizePrint_wstr(const wchar_t *s); + void NormalizePrint_wstr_Path(const wchar_t *s); }; CStdOutStream & endl(CStdOutStream & outStream) throw(); diff --git a/CPP/Common/StringConvert.cpp b/CPP/Common/StringConvert.cpp index f25396a..79ff9e0 100644 --- a/CPP/Common/StringConvert.cpp +++ b/CPP/Common/StringConvert.cpp @@ -267,8 +267,10 @@ void MultiByteToUnicodeString2(UString &dest, const AString &src, UINT codePage) if (codePage == CP_UTF8 || g_ForceToUTF8) { +#if 1 ConvertUTF8ToUnicode(src, dest); return; +#endif } const size_t limit = ((size_t)src.Len() + 1) * 2; @@ -278,48 +280,47 @@ void MultiByteToUnicodeString2(UString &dest, const AString &src, UINT codePage) { dest.ReleaseBuf_SetEnd((unsigned)len); - #if WCHAR_MAX > 0xffff +#if WCHAR_MAX > 0xffff d = dest.GetBuf(); for (size_t i = 0;; i++) { - // wchar_t c = dest[i]; wchar_t c = d[i]; + // printf("\ni=%2d c = %4x\n", (unsigned)i, (unsigned)c); if (c == 0) break; if (c >= 0x10000 && c < 0x110000) { - /* - c -= 0x10000; - unsigned c0 = 0xd800 + ((c >> 10) & 0x3FF); - dest.ReplaceOneCharAtPos(i, c0); - i++; - c = 0xdc00 + (c & 0x3FF); - dest.Insert_wchar_t(i, c); - */ - UString temp = d + i; + UString tempString = d + i; + const wchar_t *t = tempString.Ptr(); - for (size_t t = 0;; t++) + for (;;) { - wchar_t w = temp[t]; + wchar_t w = *t++; + // printf("\nchar=%x\n", w); if (w == 0) break; if (i == limit) break; // unexpected error if (w >= 0x10000 && w < 0x110000) { +#if 1 if (i + 1 == limit) break; // unexpected error w -= 0x10000; - d[i++] = (unsigned)0xd800 + (((unsigned)w >> 10) & 0x3FF); - w = 0xdc00 + (w & 0x3FF); + d[i++] = (unsigned)0xd800 + (((unsigned)w >> 10) & 0x3ff); + w = 0xdc00 + (w & 0x3ff); +#else + // w = '_'; // for debug +#endif } d[i++] = w; } dest.ReleaseBuf_SetEnd((unsigned)i); + break; } } - #endif +#endif /* printf("\nMultiByteToUnicodeString2 (%d) %s\n", (int)src.Len(), src.Ptr()); @@ -395,34 +396,39 @@ static void UnicodeStringToMultiByte2(AString &dest, const UString &src2, UINT c // if (codePage == 1234567) // for debug purposes if (codePage == CP_UTF8 || g_ForceToUTF8) { +#if 1 defaultCharWasUsed = false; ConvertUnicodeToUTF8(src2, dest); return; +#endif } UString src = src2; - #if WCHAR_MAX > 0xffff +#if WCHAR_MAX > 0xffff { src.Empty(); for (unsigned i = 0; i < src2.Len();) { - wchar_t c = src2[i]; - if (c >= 0xd800 && c < 0xdc00 && i + 1 != src2.Len()) + wchar_t c = src2[i++]; + if (c >= 0xd800 && c < 0xdc00 && i != src2.Len()) { - const wchar_t c2 = src2[i + 1]; - if (c2 >= 0xdc00 && c2 < 0x10000) + const wchar_t c2 = src2[i]; + if (c2 >= 0xdc00 && c2 < 0xe000) { +#if 1 // printf("\nSurragate [%d]: %4x %4x -> ", i, (int)c, (int)c2); c = 0x10000 + ((c & 0x3ff) << 10) + (c2 & 0x3ff); // printf("%4x\n", (int)c); i++; +#else + // c = '_'; // for debug +#endif } } src += c; - i++; } } - #endif +#endif dest.Empty(); defaultCharWasUsed = false; diff --git a/CPP/Common/StringToInt.cpp b/CPP/Common/StringToInt.cpp index bc4926e..ba8a81b 100644 --- a/CPP/Common/StringToInt.cpp +++ b/CPP/Common/StringToInt.cpp @@ -2,57 +2,63 @@ #include "StdAfx.h" +#include +#if defined(_MSC_VER) && (_MSC_VER >= 1600) +#include // for WCHAR_MAX in vs2022 +#endif + #include "StringToInt.h" static const UInt32 k_UInt32_max = 0xFFFFFFFF; static const UInt64 k_UInt64_max = UINT64_CONST(0xFFFFFFFFFFFFFFFF); // static const UInt64 k_UInt64_max = (UInt64)(Int64)-1; +#define DIGIT_TO_VALUE(charTypeUnsigned, digit) ((unsigned)(charTypeUnsigned)digit - '0') +// #define DIGIT_TO_VALUE(charTypeUnsigned, digit) ((unsigned)digit - '0') +// #define DIGIT_TO_VALUE(charTypeUnsigned, digit) ((unsigned)(digit - '0')) + #define CONVERT_STRING_TO_UINT_FUNC(uintType, charType, charTypeUnsigned) \ - uintType ConvertStringTo ## uintType(const charType *s, const charType **end) throw() { \ +uintType ConvertStringTo ## uintType(const charType *s, const charType **end) throw() { \ + if (end) *end = s; \ + uintType res = 0; \ + for (;; s++) { \ + const unsigned v = DIGIT_TO_VALUE(charTypeUnsigned, *s); \ + if (v > 9) { if (end) *end = s; return res; } \ + if (res > (k_ ## uintType ## _max) / 10) return 0; \ + res *= 10; \ + if (res > (k_ ## uintType ## _max) - v) return 0; \ + res += v; }} + +// arm-linux-gnueabi GCC compilers give (WCHAR_MAX > UINT_MAX) by some unknown reason +// so we don't use this branch +#if 0 && WCHAR_MAX > UINT_MAX +/* + if (sizeof(wchar_t) > sizeof(unsigned) + we must use CONVERT_STRING_TO_UINT_FUNC_SLOW + But we just stop compiling instead. + We need some real cases to test this code. +*/ +#error Stop_Compiling_WCHAR_MAX_IS_LARGER_THAN_UINT_MAX +#define CONVERT_STRING_TO_UINT_FUNC_SLOW(uintType, charType, charTypeUnsigned) \ +uintType ConvertStringTo ## uintType(const charType *s, const charType **end) throw() { \ if (end) *end = s; \ uintType res = 0; \ for (;; s++) { \ - charTypeUnsigned c = (charTypeUnsigned)*s; \ + const charTypeUnsigned c = (charTypeUnsigned)*s; \ if (c < '0' || c > '9') { if (end) *end = s; return res; } \ if (res > (k_ ## uintType ## _max) / 10) return 0; \ res *= 10; \ - unsigned v = (unsigned)(c - '0'); \ + const unsigned v = (unsigned)(c - '0'); \ if (res > (k_ ## uintType ## _max) - v) return 0; \ res += v; }} +#endif + CONVERT_STRING_TO_UINT_FUNC(UInt32, char, Byte) CONVERT_STRING_TO_UINT_FUNC(UInt32, wchar_t, wchar_t) CONVERT_STRING_TO_UINT_FUNC(UInt64, char, Byte) CONVERT_STRING_TO_UINT_FUNC(UInt64, wchar_t, wchar_t) -/* -Int32 ConvertStringToInt32(const char *s, const char **end) throw() -{ - if (end) - *end = s; - const char *s2 = s; - if (*s == '-') - s2++; - if (*s2 == 0) - return 0; - const char *end2; - UInt32 res = ConvertStringToUInt32(s2, &end2); - if (*s == '-') - { - if (res > ((UInt32)1 << (32 - 1))) - return 0; - } - else if ((res & ((UInt32)1 << (32 - 1))) != 0) - return 0; - if (end) - *end = end2; - if (*s == '-') - return -(Int32)res; - return (Int32)res; -} -*/ - Int32 ConvertStringToInt32(const wchar_t *s, const wchar_t **end) throw() { if (end) @@ -60,112 +66,89 @@ Int32 ConvertStringToInt32(const wchar_t *s, const wchar_t **end) throw() const wchar_t *s2 = s; if (*s == '-') s2++; - if (*s2 == 0) - return 0; const wchar_t *end2; UInt32 res = ConvertStringToUInt32(s2, &end2); - if (*s == '-') + if (s2 == end2) + return 0; + if (s != s2) { - if (res > ((UInt32)1 << (32 - 1))) + if (res > (UInt32)1 << (32 - 1)) + return 0; + res = 0 - res; + } + else + { + if (res & (UInt32)1 << (32 - 1)) return 0; } - else if ((res & ((UInt32)1 << (32 - 1))) != 0) - return 0; if (end) *end = end2; - if (*s == '-') - return -(Int32)res; return (Int32)res; } -UInt32 ConvertOctStringToUInt32(const char *s, const char **end) throw() -{ - if (end) - *end = s; - UInt32 res = 0; - for (;; s++) - { - unsigned c = (unsigned char)*s; - if (c < '0' || c > '7') - { - if (end) - *end = s; - return res; - } - if ((res & (UInt32)7 << (32 - 3)) != 0) - return 0; - res <<= 3; - res |= (unsigned)(c - '0'); - } + +#define CONVERT_OCT_STRING_TO_UINT_FUNC(uintType) \ +uintType ConvertOctStringTo ## uintType(const char *s, const char **end) throw() \ +{ \ + if (end) *end = s; \ + uintType res = 0; \ + for (;; s++) { \ + const unsigned c = (unsigned)(Byte)*s - '0'; \ + if (c > 7) { \ + if (end) \ + *end = s; \ + return res; \ + } \ + if (res & (uintType)7 << (sizeof(uintType) * 8 - 3)) \ + return 0; \ + res <<= 3; \ + res |= c; \ + } \ } -UInt64 ConvertOctStringToUInt64(const char *s, const char **end) throw() -{ - if (end) - *end = s; - UInt64 res = 0; - for (;; s++) - { - unsigned c = (unsigned char)*s; - if (c < '0' || c > '7') - { - if (end) - *end = s; - return res; - } - if ((res & (UInt64)7 << (64 - 3)) != 0) - return 0; - res <<= 3; - res |= (unsigned)(c - '0'); - } +CONVERT_OCT_STRING_TO_UINT_FUNC(UInt32) +CONVERT_OCT_STRING_TO_UINT_FUNC(UInt64) + + +#define CONVERT_HEX_STRING_TO_UINT_FUNC(uintType) \ +uintType ConvertHexStringTo ## uintType(const char *s, const char **end) throw() \ +{ \ + if (end) *end = s; \ + uintType res = 0; \ + for (;; s++) { \ + unsigned c = (unsigned)(Byte)*s; \ + Z7_PARSE_HEX_DIGIT(c, { if (end) *end = s; return res; }) \ + if (res & (uintType)0xF << (sizeof(uintType) * 8 - 4)) \ + return 0; \ + res <<= 4; \ + res |= c; \ + } \ } -UInt32 ConvertHexStringToUInt32(const char *s, const char **end) throw() +CONVERT_HEX_STRING_TO_UINT_FUNC(UInt32) +CONVERT_HEX_STRING_TO_UINT_FUNC(UInt64) + +const char *FindNonHexChar(const char *s) throw() { - if (end) - *end = s; - UInt32 res = 0; - for (;; s++) + for (;;) { - unsigned c = (Byte)*s; - unsigned v; - if (c >= '0' && c <= '9') v = (c - '0'); - else if (c >= 'A' && c <= 'F') v = 10 + (c - 'A'); - else if (c >= 'a' && c <= 'f') v = 10 + (c - 'a'); - else - { - if (end) - *end = s; - return res; - } - if ((res & (UInt32)0xF << (32 - 4)) != 0) - return 0; - res <<= 4; - res |= v; + unsigned c = (Byte)*s++; // pointer can go 1 byte after end + c -= '0'; + if (c <= 9) + continue; + c -= 'A' - '0'; + c &= ~0x20u; + if (c > 5) + return s - 1; } } -UInt64 ConvertHexStringToUInt64(const char *s, const char **end) throw() +Byte *ParseHexString(const char *s, Byte *dest) throw() { - if (end) - *end = s; - UInt64 res = 0; - for (;; s++) + for (;;) { - unsigned c = (Byte)*s; - unsigned v; - if (c >= '0' && c <= '9') v = (c - '0'); - else if (c >= 'A' && c <= 'F') v = 10 + (c - 'A'); - else if (c >= 'a' && c <= 'f') v = 10 + (c - 'a'); - else - { - if (end) - *end = s; - return res; - } - if ((res & (UInt64)0xF << (64 - 4)) != 0) - return 0; - res <<= 4; - res |= v; + unsigned v0 = (Byte)s[0]; Z7_PARSE_HEX_DIGIT(v0, return dest;) + unsigned v1 = (Byte)s[1]; s += 2; Z7_PARSE_HEX_DIGIT(v1, return dest;) + *dest++ = (Byte)(v1 | (v0 << 4)); } } diff --git a/CPP/Common/StringToInt.h b/CPP/Common/StringToInt.h index c9dce7d..31ab274 100644 --- a/CPP/Common/StringToInt.h +++ b/CPP/Common/StringToInt.h @@ -19,4 +19,20 @@ UInt64 ConvertOctStringToUInt64(const char *s, const char **end) throw(); UInt32 ConvertHexStringToUInt32(const char *s, const char **end) throw(); UInt64 ConvertHexStringToUInt64(const char *s, const char **end) throw(); +#define Z7_PARSE_HEX_DIGIT(c, err_op) \ +{ c -= '0'; \ + if (c > 9) { \ + c -= 'A' - '0'; \ + c &= ~0x20u; \ + if (c > 5) { err_op } \ + c += 10; \ + } \ +} + +const char *FindNonHexChar(const char *s) throw(); + +// in: (dest != NULL) +// returns: pointer in dest array after last written byte +Byte *ParseHexString(const char *s, Byte *dest) throw(); + #endif diff --git a/CPP/Common/TextConfig.cpp b/CPP/Common/TextConfig.cpp index d3e561c..5714e46 100644 --- a/CPP/Common/TextConfig.cpp +++ b/CPP/Common/TextConfig.cpp @@ -90,15 +90,14 @@ bool GetTextConfig(const AString &s, CObjectVector &pairs) c = s[pos++]; switch (c) { - case 'n': message += '\n'; break; - case 't': message += '\t'; break; - case '\\': message += '\\'; break; - case '\"': message += '\"'; break; - default: message += '\\'; message += c; break; + case 'n': c = '\n'; break; + case 't': c = '\t'; break; + case '\\': break; + case '\"': break; + default: message += '\\'; break; } } - else - message += c; + message += c; } if (!ConvertUTF8ToUnicode(message, pair.String)) return false; diff --git a/CPP/Common/UTFConvert.cpp b/CPP/Common/UTFConvert.cpp index fb166b7..509de76 100644 --- a/CPP/Common/UTFConvert.cpp +++ b/CPP/Common/UTFConvert.cpp @@ -135,7 +135,7 @@ we can place 128 ESCAPE chars to #endif #define IS_SURROGATE_POINT(v) (((v) & (UInt32)0xfffff800) == 0xd800) -#define IS_LOW_SURROGATE_POINT(v) (((v) & (UInt32)0xfffffC00) == 0xdc00) +#define IS_LOW_SURROGATE_POINT(v) (((v) & (UInt32)0xfffffc00) == 0xdc00) #define UTF_ERROR_UTF8_CHECK \ @@ -168,14 +168,14 @@ void CUtf8Check::Check_Buf(const char *src, size_t size) throw() if (c < 0x80) continue; - if (c < 0xc0 + 2) // it's limit for 0x140000 unicode codes : win32 compatibility + if (c < 0xc0 + 2) UTF_ERROR_UTF8_CHECK unsigned numBytes; - UInt32 val = c; MY_UTF8_HEAD_PARSE2(1) else MY_UTF8_HEAD_PARSE2(2) + else MY_UTF8_HEAD_PARSE2(3) else MY_UTF8_HEAD_PARSE2(4) else MY_UTF8_HEAD_PARSE2(5) else @@ -768,7 +768,7 @@ void Convert_UTF16_To_UTF32(const UString &src, UString &dest) if (c >= 0xd800 && c < 0xdc00 && i < src.Len()) { const wchar_t c2 = src[i]; - if (c2 >= 0xdc00 && c2 < 0x10000) + if (c2 >= 0xdc00 && c2 < 0xe000) { // printf("\nSurragate [%d]: %4x %4x -> ", i, (int)c, (int)c2); c = 0x10000 + ((c & 0x3ff) << 10) + (c2 & 0x3ff); diff --git a/CPP/Common/Xxh64Reg.cpp b/CPP/Common/Xxh64Reg.cpp new file mode 100644 index 0000000..8abd15e --- /dev/null +++ b/CPP/Common/Xxh64Reg.cpp @@ -0,0 +1,97 @@ +// Xxh64Reg.cpp + +#include "StdAfx.h" + +#include "../../C/Xxh64.h" +#include "../../C/CpuArch.h" + +// #define Z7_USE_HHX64_ORIGINAL +#ifdef Z7_USE_HHX64_ORIGINAL +#ifdef __clang__ +#include "../../C/zstd7z/7z_zstd_cmpl.h" +#pragma GCC diagnostic ignored "-Wlanguage-extension-token" +#endif +#define XXH_STATIC_LINKING_ONLY +#include "../../C/zstd7z/common/xxhash.h" +#endif + +#include "../Common/MyCom.h" + +#include "../7zip/Common/RegisterCodec.h" + +Z7_CLASS_IMP_COM_1( + CXxh64Hasher + , IHasher +) + CXxh64 _state; +public: + Byte _mtDummy[1 << 7]; // it's public to eliminate clang warning: unused private field + CXxh64Hasher() { Init(); } +}; + +Z7_COM7F_IMF2(void, CXxh64Hasher::Init()) +{ + Xxh64_Init(&_state); +} + +Z7_COM7F_IMF2(void, CXxh64Hasher::Update(const void *data, UInt32 size)) +{ +#if 1 + Xxh64_Update(&_state, data, size); +#else // for debug: + for (;;) + { + if (size == 0) + return; + UInt32 size2 = (size * 0x85EBCA87) % size / 8; + if (size2 == 0) + size2 = 1; + Xxh64_Update(&_state, data, size2); + data = (const void *)((const Byte *)data + size2); + size -= size2; + } +#endif +} + +Z7_COM7F_IMF2(void, CXxh64Hasher::Final(Byte *digest)) +{ + const UInt64 val = Xxh64_Digest(&_state); + SetUi64(digest, val) +} + +REGISTER_HASHER(CXxh64Hasher, 0x211, "XXH64", 8) + + + +#ifdef Z7_USE_HHX64_ORIGINAL +namespace NOriginal +{ +Z7_CLASS_IMP_COM_1( + CXxh64Hasher + , IHasher +) + XXH64_state_t _state; +public: + Byte _mtDummy[1 << 7]; // it's public to eliminate clang warning: unused private field + CXxh64Hasher() { Init(); } +}; + +Z7_COM7F_IMF2(void, CXxh64Hasher::Init()) +{ + XXH64_reset(&_state, 0); +} + +Z7_COM7F_IMF2(void, CXxh64Hasher::Update(const void *data, UInt32 size)) +{ + XXH64_update(&_state, data, size); +} + +Z7_COM7F_IMF2(void, CXxh64Hasher::Final(Byte *digest)) +{ + const UInt64 val = XXH64_digest(&_state); + SetUi64(digest, val) +} + +REGISTER_HASHER(CXxh64Hasher, 0x212, "XXH64a", 8) +} +#endif diff --git a/CPP/Windows/Control/Dialog.cpp b/CPP/Windows/Control/Dialog.cpp index c8f1bd2..cbb000b 100644 --- a/CPP/Windows/Control/Dialog.cpp +++ b/CPP/Windows/Control/Dialog.cpp @@ -93,6 +93,7 @@ bool CDialog::OnButtonClicked(unsigned buttonID, HWND /* buttonHWND */) case IDOK: OnOK(); break; case IDCANCEL: OnCancel(); break; case IDCLOSE: OnClose(); break; + case IDCONTINUE: OnContinue(); break; case IDHELP: OnHelp(); break; default: return false; } diff --git a/CPP/Windows/Control/Dialog.h b/CPP/Windows/Control/Dialog.h index 06be4bf..2510c8f 100644 --- a/CPP/Windows/Control/Dialog.h +++ b/CPP/Windows/Control/Dialog.h @@ -8,6 +8,10 @@ namespace NWindows { namespace NControl { +#ifndef IDCONTINUE +#define IDCONTINUE 11 +#endif + class CDialog: public CWindow { // Z7_CLASS_NO_COPY(CDialog) @@ -64,14 +68,26 @@ class CDialog: public CWindow return window.GetText(s); } +/* bool SetItemInt(unsigned itemID, UINT value, bool isSigned) { return BOOLToBool(SetDlgItemInt(_window, (int)itemID, value, BoolToBOOL(isSigned))); } +*/ + bool SetItemUInt(unsigned itemID, UINT value) + { return BOOLToBool(SetDlgItemInt(_window, (int)itemID, value, FALSE)); } +/* bool GetItemInt(unsigned itemID, bool isSigned, UINT &value) { BOOL result; value = GetDlgItemInt(_window, (int)itemID, &result, BoolToBOOL(isSigned)); return BOOLToBool(result); } +*/ + bool GetItemUInt(unsigned itemID, UINT &value) + { + BOOL result; + value = GetDlgItemInt(_window, (int)itemID, &result, FALSE); + return BOOLToBool(result); + } HWND GetNextGroupItem(HWND control, bool previous) { return GetNextDlgGroupItem(_window, control, BoolToBOOL(previous)); } @@ -126,6 +142,7 @@ class CDialog: public CWindow virtual bool OnButtonClicked(unsigned buttonID, HWND buttonHWND); virtual void OnOK() {} + virtual void OnContinue() {} virtual void OnCancel() {} virtual void OnClose() {} virtual bool OnNotify(UINT /* controlID */, LPNMHDR /* lParam */) { return false; } @@ -157,6 +174,7 @@ class CModelessDialog: public CDialog bool Create(LPCWSTR templateName, HWND parentWindow); #endif virtual void OnOK() Z7_override { Destroy(); } + virtual void OnContinue() Z7_override { Destroy(); } virtual void OnCancel() Z7_override { Destroy(); } virtual void OnClose() Z7_override { Destroy(); } }; @@ -172,6 +190,7 @@ class CModalDialog: public CDialog bool End(INT_PTR result) { return BOOLToBool(::EndDialog(_window, result)); } virtual void OnOK() Z7_override { End(IDOK); } + virtual void OnContinue() Z7_override { End(IDCONTINUE); } virtual void OnCancel() Z7_override { End(IDCANCEL); } virtual void OnClose() Z7_override { End(IDCLOSE); } }; diff --git a/CPP/Windows/Control/PropertyPage.cpp b/CPP/Windows/Control/PropertyPage.cpp index f8effe6..9b36cbe 100644 --- a/CPP/Windows/Control/PropertyPage.cpp +++ b/CPP/Windows/Control/PropertyPage.cpp @@ -141,7 +141,7 @@ INT_PTR MyPropertySheet(const CObjectVector &pagesInfo, HWND hwndPare sheet.pszCaption = titleA; sheet.nPages = pagesA.Size(); sheet.nStartPage = 0; - sheet.ppsp = (LPCPROPSHEETPAGEA)(const void *)&pagesA.Front(); + sheet.ppsp = (LPCPROPSHEETPAGEA)(const void *)pagesA.ConstData(); sheet.pfnCallback = NULL; return ::PropertySheetA(&sheet); } @@ -156,7 +156,7 @@ INT_PTR MyPropertySheet(const CObjectVector &pagesInfo, HWND hwndPare sheet.pszCaption = title; sheet.nPages = pagesW.Size(); sheet.nStartPage = 0; - sheet.ppsp = (LPCPROPSHEETPAGEW)(const void *)&pagesW.Front(); + sheet.ppsp = (LPCPROPSHEETPAGEW)(const void *)pagesW.ConstData(); sheet.pfnCallback = NULL; return ::PropertySheetW(&sheet); } diff --git a/CPP/Windows/DLL.cpp b/CPP/Windows/DLL.cpp index b2499ec..025df17 100644 --- a/CPP/Windows/DLL.cpp +++ b/CPP/Windows/DLL.cpp @@ -61,14 +61,14 @@ bool CLibrary::Load(CFSTR path) throw() bool MyGetModuleFileName(FString &path) { - HMODULE hModule = g_hInstance; + const HMODULE hModule = g_hInstance; path.Empty(); #ifndef _UNICODE if (!g_IsNT) { TCHAR s[MAX_PATH + 2]; s[0] = 0; - DWORD size = ::GetModuleFileName(hModule, s, MAX_PATH + 1); + const DWORD size = ::GetModuleFileName(hModule, s, MAX_PATH + 1); if (size <= MAX_PATH && size != 0) { path = fas2fs(s); @@ -80,7 +80,7 @@ bool MyGetModuleFileName(FString &path) { WCHAR s[MAX_PATH + 2]; s[0] = 0; - DWORD size = ::GetModuleFileNameW(hModule, s, MAX_PATH + 1); + const DWORD size = ::GetModuleFileNameW(hModule, s, MAX_PATH + 1); if (size <= MAX_PATH && size != 0) { path = us2fs(s); @@ -97,7 +97,7 @@ FString GetModuleDirPrefix() FString s; if (MyGetModuleFileName(s)) { - int pos = s.ReverseFind_PathSepar(); + const int pos = s.ReverseFind_PathSepar(); if (pos >= 0) s.DeleteFrom((unsigned)(pos + 1)); } @@ -114,7 +114,6 @@ FString GetModuleDirPrefix() #include #include -#include "../Common/Common.h" // FARPROC void *GetProcAddress(HMODULE module, LPCSTR procName) diff --git a/CPP/Windows/FileDir.cpp b/CPP/Windows/FileDir.cpp index 5b1f340..dfeed82 100644 --- a/CPP/Windows/FileDir.cpp +++ b/CPP/Windows/FileDir.cpp @@ -37,7 +37,12 @@ static bool FiTime_To_timespec(const CFiTime *ft, timespec &ts) { if (ft) { +#if defined(_AIX) + ts.tv_sec = ft->tv_sec; + ts.tv_nsec = ft->tv_nsec; +#else ts = *ft; +#endif return true; } // else @@ -246,6 +251,11 @@ bool MyMoveFile(CFSTR oldFile, CFSTR newFile) } #ifndef UNDER_CE +#if !defined(Z7_WIN32_WINNT_MIN) || Z7_WIN32_WINNT_MIN < 0x0500 // Win2000 +#define Z7_USE_DYN_CreateHardLink +#endif + +#ifdef Z7_USE_DYN_CreateHardLink EXTERN_C_BEGIN typedef BOOL (WINAPI *Func_CreateHardLinkW)( LPCWSTR lpFileName, @@ -253,6 +263,7 @@ typedef BOOL (WINAPI *Func_CreateHardLinkW)( LPSECURITY_ATTRIBUTES lpSecurityAttributes ); EXTERN_C_END +#endif #endif // UNDER_CE bool MyCreateHardLink(CFSTR newFileName, CFSTR existFileName) @@ -270,6 +281,7 @@ bool MyCreateHardLink(CFSTR newFileName, CFSTR existFileName) else #endif { +#ifdef Z7_USE_DYN_CreateHardLink const Func_CreateHardLinkW my_CreateHardLinkW = Z7_GET_PROC_ADDRESS( @@ -277,9 +289,13 @@ bool MyCreateHardLink(CFSTR newFileName, CFSTR existFileName) "CreateHardLinkW"); if (!my_CreateHardLinkW) return false; + #define MY_CreateHardLinkW my_CreateHardLinkW +#else + #define MY_CreateHardLinkW CreateHardLinkW +#endif IF_USE_MAIN_PATH_2(newFileName, existFileName) { - if (my_CreateHardLinkW(fs2us(newFileName), fs2us(existFileName), NULL)) + if (MY_CreateHardLinkW(fs2us(newFileName), fs2us(existFileName), NULL)) return true; } #ifdef Z7_LONG_PATH @@ -287,7 +303,7 @@ bool MyCreateHardLink(CFSTR newFileName, CFSTR existFileName) { UString d1, d2; if (GetSuperPaths(newFileName, existFileName, d1, d2, USE_MAIN_PATH_2)) - return BOOLToBool(my_CreateHardLinkW(d1, d2, NULL)); + return BOOLToBool(MY_CreateHardLinkW(d1, d2, NULL)); } #endif } @@ -780,7 +796,7 @@ bool CreateTempFile2(CFSTR prefix, bool addRandom, AString &postfix, NIO::COutFi unsigned k; for (k = 0; k < 8; k++) { - const unsigned t = val & 0xF; + const unsigned t = (unsigned)val & 0xF; val >>= 4; s[k] = (char)((t < 10) ? ('0' + t) : ('A' + (t - 10))); } @@ -805,7 +821,7 @@ bool CreateTempFile2(CFSTR prefix, bool addRandom, AString &postfix, NIO::COutFi } if (outFile) { - if (outFile->Create(path, false)) + if (outFile->Create_NEW(path)) return true; } else @@ -928,7 +944,7 @@ bool RemoveDir(CFSTR path) static BOOL My_CopyFile(CFSTR oldFile, CFSTR newFile) { NWindows::NFile::NIO::COutFile outFile; - if (!outFile.Create(newFile, false)) + if (!outFile.Create_NEW(newFile)) return FALSE; NWindows::NFile::NIO::CInFile inFile; diff --git a/CPP/Windows/FileFind.cpp b/CPP/Windows/FileFind.cpp index c562a90..ca387f6 100644 --- a/CPP/Windows/FileFind.cpp +++ b/CPP/Windows/FileFind.cpp @@ -25,6 +25,14 @@ using namespace NName; #if defined(_WIN32) && !defined(UNDER_CE) +#if !defined(Z7_WIN32_WINNT_MIN) || Z7_WIN32_WINNT_MIN < 0x0502 // Win2003 +#define Z7_USE_DYN_FindFirstStream +#endif + +#ifdef Z7_USE_DYN_FindFirstStream + +Z7_DIAGNOSTIC_IGNORE_CAST_FUNCTION + EXTERN_C_BEGIN typedef enum @@ -46,6 +54,12 @@ typedef BOOL (APIENTRY *Func_FindNextStreamW)(HANDLE findStream, LPVOID findStre EXTERN_C_END +#else + +#define MY_WIN32_FIND_STREAM_DATA WIN32_FIND_STREAM_DATA +#define My_FindStreamInfoStandard FindStreamInfoStandard + +#endif #endif // defined(_WIN32) && !defined(UNDER_CE) @@ -95,6 +109,86 @@ void CFileInfoBase::ClearBase() throw() #endif } + +bool CFileInfoBase::SetAs_StdInFile() +{ + ClearBase(); + Size = (UInt64)(Int64)-1; + NTime::GetCurUtc_FiTime(MTime); + CTime = ATime = MTime; + +#ifdef _WIN32 + + /* in GUI mode : GetStdHandle(STD_INPUT_HANDLE) returns NULL, + and it doesn't set LastError. */ +#if 1 + SetLastError(0); + const HANDLE h = GetStdHandle(STD_INPUT_HANDLE); + if (!h || h == INVALID_HANDLE_VALUE) + { + if (GetLastError() == 0) + SetLastError(ERROR_INVALID_HANDLE); + return false; + } + BY_HANDLE_FILE_INFORMATION info; + if (GetFileInformationByHandle(h, &info) + && info.dwVolumeSerialNumber) + { + Size = (((UInt64)info.nFileSizeHigh) << 32) + info.nFileSizeLow; + // FileID_Low = (((UInt64)info.nFileIndexHigh) << 32) + info.nFileIndexLow; + // NumLinks = SupportHardLinks ? info.nNumberOfLinks : 1; + Attrib = info.dwFileAttributes; + CTime = info.ftCreationTime; + ATime = info.ftLastAccessTime; + MTime = info.ftLastWriteTime; + } +#if 0 + printf( + "\ndwFileAttributes = %8x" + "\nftCreationTime = %8x" + "\nftLastAccessTime = %8x" + "\nftLastWriteTime = %8x" + "\ndwVolumeSerialNumber = %8x" + "\nnFileSizeHigh = %8x" + "\nnFileSizeLow = %8x" + "\nnNumberOfLinks = %8x" + "\nnFileIndexHigh = %8x" + "\nnFileIndexLow = %8x \n", + (unsigned)info.dwFileAttributes, + (unsigned)info.ftCreationTime.dwHighDateTime, + (unsigned)info.ftLastAccessTime.dwHighDateTime, + (unsigned)info.ftLastWriteTime.dwHighDateTime, + (unsigned)info.dwVolumeSerialNumber, + (unsigned)info.nFileSizeHigh, + (unsigned)info.nFileSizeLow, + (unsigned)info.nNumberOfLinks, + (unsigned)info.nFileIndexHigh, + (unsigned)info.nFileIndexLow); +#endif +#endif + +#else // non-Wiondow + + mode = S_IFIFO | 0777; // 0755 : 0775 : 0664 : 0644 : +#if 1 + struct stat st; + if (fstat(0, &st) == 0) + { + SetFrom_stat(st); + if (!S_ISREG(st.st_mode) + // S_ISFIFO(st->st_mode) + || st.st_size == 0) + { + Size = (UInt64)(Int64)-1; + // mode = S_IFIFO | 0777; + } + } +#endif +#endif + + return true; +} + bool CFileInfo::IsDots() const throw() { if (!IsDir() || Name.IsEmpty()) @@ -252,9 +346,11 @@ bool CFindFile::FindNext(CFileInfo &fi) //////////////////////////////// // AltStreams +#ifdef Z7_USE_DYN_FindFirstStream static Func_FindFirstStreamW g_FindFirstStreamW; -static Func_FindNextStreamW g_FindNextStreamW; - +static Func_FindNextStreamW g_FindNextStreamW; +#define MY_FindFirstStreamW g_FindFirstStreamW +#define MY_FindNextStreamW g_FindNextStreamW static struct CFindStreamLoader { CFindStreamLoader() @@ -268,6 +364,11 @@ static struct CFindStreamLoader "FindNextStreamW"); } } g_FindStreamLoader; +#else +#define MY_FindFirstStreamW FindFirstStreamW +#define MY_FindNextStreamW FindNextStreamW +#endif + bool CStreamInfo::IsMainStream() const throw() { @@ -320,16 +421,18 @@ bool CFindStream::FindFirst(CFSTR path, CStreamInfo &si) { if (!Close()) return false; +#ifdef Z7_USE_DYN_FindFirstStream if (!g_FindFirstStreamW) { ::SetLastError(ERROR_CALL_NOT_IMPLEMENTED); return false; } +#endif { MY_WIN32_FIND_STREAM_DATA sd; SetLastError(0); IF_USE_MAIN_PATH - _handle = g_FindFirstStreamW(fs2us(path), My_FindStreamInfoStandard, &sd, 0); + _handle = MY_FindFirstStreamW(fs2us(path), My_FindStreamInfoStandard, &sd, 0); if (_handle == INVALID_HANDLE_VALUE) { if (::GetLastError() == ERROR_HANDLE_EOF) @@ -340,7 +443,7 @@ bool CFindStream::FindFirst(CFSTR path, CStreamInfo &si) { UString superPath; if (GetSuperPath(path, superPath, USE_MAIN_PATH)) - _handle = g_FindFirstStreamW(superPath, My_FindStreamInfoStandard, &sd, 0); + _handle = MY_FindFirstStreamW(superPath, My_FindStreamInfoStandard, &sd, 0); } #endif } @@ -353,14 +456,16 @@ bool CFindStream::FindFirst(CFSTR path, CStreamInfo &si) bool CFindStream::FindNext(CStreamInfo &si) { +#ifdef Z7_USE_DYN_FindFirstStream if (!g_FindNextStreamW) { ::SetLastError(ERROR_CALL_NOT_IMPLEMENTED); return false; } +#endif { MY_WIN32_FIND_STREAM_DATA sd; - if (!g_FindNextStreamW(_handle, &sd)) + if (!MY_FindNextStreamW(_handle, &sd)) return false; Convert_WIN32_FIND_STREAM_DATA_to_StreamInfo(sd, si); } @@ -622,7 +727,7 @@ bool CFileInfo::Find(CFSTR path, bool followLink) FString s (path); s.Add_PathSepar(); - s += '*'; // CHAR_ANY_MASK + s.Add_Char('*'); // CHAR_ANY_MASK bool isOK = false; if (finder.FindFirst(s, *this)) { @@ -636,7 +741,7 @@ bool CFileInfo::Find(CFSTR path, bool followLink) But it's possible that there are another items */ } { - DWORD attrib = GetFileAttrib(path); + const DWORD attrib = GetFileAttrib(path); if (isOK || (attrib != INVALID_FILE_ATTRIBUTES && (attrib & FILE_ATTRIBUTE_DIRECTORY) != 0)) { ClearBase(); @@ -763,7 +868,7 @@ bool DoesFileOrDirExist(CFSTR name) void CEnumerator::SetDirPrefix(const FString &dirPrefix) { _wildcard = dirPrefix; - _wildcard += '*'; + _wildcard.Add_Char('*'); } bool CEnumerator::NextAny(CFileInfo &fi) @@ -925,7 +1030,7 @@ bool MyGetLogicalDriveStrings(CObjectVector &driveStrings) // ---------- POSIX ---------- -static int MY__lstat(CFSTR path, struct stat *st, bool followLink) +static int MY_lstat(CFSTR path, struct stat *st, bool followLink) { memset(st, 0, sizeof(*st)); int res; @@ -941,18 +1046,26 @@ static int MY__lstat(CFSTR path, struct stat *st, bool followLink) // printf("\nstat\n"); res = stat(path, st); } - /* - printf("\nres = %d\n", res); - printf("\n st_dev = %lld \n", (long long)(st->st_dev)); - printf("\n st_ino = %lld \n", (long long)(st->st_ino)); - printf("\n st_mode = %lld \n", (long long)(st->st_mode)); - printf("\n st_nlink = %lld \n", (long long)(st->st_nlink)); - printf("\n st_uid = %lld \n", (long long)(st->st_uid)); - printf("\n st_gid = %lld \n", (long long)(st->st_gid)); - printf("\n st_size = %lld \n", (long long)(st->st_size)); - printf("\n st_blksize = %lld \n", (long long)(st->st_blksize)); - printf("\n st_blocks = %lld \n", (long long)(st->st_blocks)); - */ +#if 0 +#if defined(__clang__) && __clang_major__ >= 14 + #pragma GCC diagnostic ignored "-Wc++98-compat-pedantic" +#endif + + printf("\n st_dev = %lld", (long long)(st->st_dev)); + printf("\n st_ino = %lld", (long long)(st->st_ino)); + printf("\n st_mode = %llx", (long long)(st->st_mode)); + printf("\n st_nlink = %lld", (long long)(st->st_nlink)); + printf("\n st_uid = %lld", (long long)(st->st_uid)); + printf("\n st_gid = %lld", (long long)(st->st_gid)); + printf("\n st_size = %lld", (long long)(st->st_size)); + printf("\n st_blksize = %lld", (long long)(st->st_blksize)); + printf("\n st_blocks = %lld", (long long)(st->st_blocks)); + printf("\n st_ctim = %lld", (long long)(ST_CTIME((*st)).tv_sec)); + printf("\n st_mtim = %lld", (long long)(ST_MTIME((*st)).tv_sec)); + printf("\n st_atim = %lld", (long long)(ST_ATIME((*st)).tv_sec)); + printf(S_ISFIFO(st->st_mode) ? "\n FIFO" : "\n NO FIFO"); + printf("\n"); +#endif return res; } @@ -1006,7 +1119,7 @@ UInt32 Get_WinAttrib_From_stat(const struct stat &st) } */ -void CFileInfo::SetFrom_stat(const struct stat &st) +void CFileInfoBase::SetFrom_stat(const struct stat &st) { // IsDevice = false; @@ -1114,7 +1227,7 @@ int Uid_To_Uname(uid_t uid, AString &name) bool CFileInfo::Find_DontFill_Name(CFSTR path, bool followLink) { struct stat st; - if (MY__lstat(path, &st, followLink) != 0) + if (MY_lstat(path, &st, followLink) != 0) return false; // printf("\nFind_DontFill_Name : name=%s\n", path); SetFrom_stat(st); @@ -1145,7 +1258,7 @@ bool DoesFileExist_Raw(CFSTR name) { // FIXME for symbolic links. struct stat st; - if (MY__lstat(name, &st, false) != 0) + if (MY_lstat(name, &st, false) != 0) return false; return !S_ISDIR(st.st_mode); } @@ -1154,7 +1267,7 @@ bool DoesFileExist_FollowLink(CFSTR name) { // FIXME for symbolic links. struct stat st; - if (MY__lstat(name, &st, true) != 0) + if (MY_lstat(name, &st, true) != 0) return false; return !S_ISDIR(st.st_mode); } @@ -1162,7 +1275,7 @@ bool DoesFileExist_FollowLink(CFSTR name) bool DoesDirExist(CFSTR name, bool followLink) { struct stat st; - if (MY__lstat(name, &st, followLink) != 0) + if (MY_lstat(name, &st, followLink) != 0) return false; return S_ISDIR(st.st_mode); } @@ -1170,7 +1283,7 @@ bool DoesDirExist(CFSTR name, bool followLink) bool DoesFileOrDirExist(CFSTR name) { struct stat st; - if (MY__lstat(name, &st, false) != 0) + if (MY_lstat(name, &st, false) != 0) return false; return true; } @@ -1192,10 +1305,10 @@ bool CDirEntry::IsDots() const throw() /* some systems (like CentOS 7.x on XFS) have (Type == DT_UNKNOWN) we can call fstatat() for that case, but we use only (Name) check here */ - #if !defined(_AIX) +#if !defined(_AIX) && !defined(__sun) if (Type != DT_DIR && Type != DT_UNKNOWN) return false; - #endif +#endif return Name.Len() != 0 && Name.Len() <= 2 @@ -1232,7 +1345,7 @@ bool CEnumerator::NextAny(CDirEntry &fi, bool &found) fi.iNode = de->d_ino; - #if !defined(_AIX) +#if !defined(_AIX) && !defined(__sun) fi.Type = de->d_type; /* some systems (like CentOS 7.x on XFS) have (Type == DT_UNKNOWN) we can set (Type) from fstatat() in that case. @@ -1247,7 +1360,7 @@ bool CEnumerator::NextAny(CDirEntry &fi, bool &found) fi.Type = DT_DIR; } */ - #endif +#endif /* if (de->d_type == DT_DIR) @@ -1313,7 +1426,7 @@ bool CEnumerator::Fill_FileInfo(const CDirEntry &de, CFileInfo &fileInfo, bool f /* const FString path = _wildcard + s; - int res = MY__lstat(path, &st, followLink); + int res = MY_lstat(path, &st, followLink); */ if (res != 0) diff --git a/CPP/Windows/FileFind.h b/CPP/Windows/FileFind.h index 11408d0..f68673a 100644 --- a/CPP/Windows/FileFind.h +++ b/CPP/Windows/FileFind.h @@ -99,12 +99,12 @@ class CFileInfoBase CFileInfoBase() { ClearBase(); } void ClearBase() throw(); + bool SetAs_StdInFile(); #ifdef _WIN32 bool Fill_From_ByHandleFileInfo(CFSTR path); void SetAsDir() { Attrib = FILE_ATTRIBUTE_DIRECTORY; } // |= (FILE_ATTRIBUTE_UNIX_EXTENSION + (S_IFDIR << 16)); - void SetAsFile() { Attrib = 0; } bool IsArchived() const { return MatchesMask(FILE_ATTRIBUTE_ARCHIVE); } bool IsCompressed() const { return MatchesMask(FILE_ATTRIBUTE_COMPRESSED); } @@ -132,8 +132,8 @@ class CFileInfoBase UInt32 GetWinAttrib() const { return Get_WinAttribPosix_From_PosixMode(mode); } bool IsDir() const { return S_ISDIR(mode); } - void SetAsDir() { mode = S_IFDIR; } - void SetAsFile() { mode = S_IFREG; } + void SetAsDir() { mode = S_IFDIR | 0777; } + void SetFrom_stat(const struct stat &st); bool IsReadOnly() const { @@ -173,7 +173,6 @@ struct CFileInfo: public CFileInfoBase // bool FollowReparse(CFSTR path, bool isDir); #else bool Find_DontFill_Name(CFSTR path, bool followLink = false); - void SetFrom_stat(const struct stat &st); #endif }; @@ -278,19 +277,19 @@ typedef CFileInfo CDirEntry; struct CDirEntry { ino_t iNode; - #if !defined(_AIX) +#if !defined(_AIX) && !defined(__sun) Byte Type; - #endif +#endif FString Name; /* - #if !defined(_AIX) +#if !defined(_AIX) && !defined(__sun) bool IsDir() const { // (Type == DT_UNKNOWN) on some systems return Type == DT_DIR; } - #endif +#endif */ bool IsDots() const throw(); @@ -311,12 +310,12 @@ class CEnumerator MY_UNCOPYABLE bool Fill_FileInfo(const CDirEntry &de, CFileInfo &fileInfo, bool followLink) const; bool DirEntry_IsDir(const CDirEntry &de, bool followLink) const { - #if !defined(_AIX) +#if !defined(_AIX) && !defined(__sun) if (de.Type == DT_DIR) return true; if (de.Type != DT_UNKNOWN) return false; - #endif +#endif CFileInfo fileInfo; if (Fill_FileInfo(de, fileInfo, followLink)) { diff --git a/CPP/Windows/FileIO.cpp b/CPP/Windows/FileIO.cpp index 4ecbb7e..dc4de14 100644 --- a/CPP/Windows/FileIO.cpp +++ b/CPP/Windows/FileIO.cpp @@ -141,8 +141,17 @@ bool CFileBase::Close() throw() { if (_handle == INVALID_HANDLE_VALUE) return true; - if (!::CloseHandle(_handle)) - return false; +#if 0 + if (!IsStdStream) +#endif + { + if (!::CloseHandle(_handle)) + return false; + } +#if 0 + IsStdStream = false; + IsStdPipeStream = false; +#endif _handle = INVALID_HANDLE_VALUE; return true; } @@ -457,7 +466,7 @@ bool CInFile::Open(CFSTR fileName) // for 32 MB (maybe also for 16 MB). // And message can be "Network connection was lost" -static const UInt32 kChunkSizeMax = (1 << 22); +static const UInt32 kChunkSizeMax = 1 << 22; bool CInFile::Read1(void *data, UInt32 size, UInt32 &processedSize) throw() { @@ -469,8 +478,14 @@ bool CInFile::Read1(void *data, UInt32 size, UInt32 &processedSize) throw() bool CInFile::ReadPart(void *data, UInt32 size, UInt32 &processedSize) throw() { +#if 0 + const UInt32 chunkSizeMax = (0 || IsStdStream) ? (1 << 20) : kChunkSizeMax; + if (size > chunkSizeMax) + size = chunkSizeMax; +#else if (size > kChunkSizeMax) - size = kChunkSizeMax; + size = kChunkSizeMax; +#endif return Read1(data, size, processedSize); } @@ -486,10 +501,10 @@ bool CInFile::Read(void *data, UInt32 size, UInt32 &processedSize) throw() return false; if (processedLoc == 0) return true; - data = (void *)((unsigned char *)data + processedLoc); + data = (void *)((Byte *)data + processedLoc); size -= processedLoc; } - while (size > 0); + while (size); return true; } @@ -506,29 +521,23 @@ bool CInFile::ReadFull(void *data, size_t size, size_t &processedSize) throw() return false; if (processedLoc == 0) return true; - data = (void *)((unsigned char *)data + processedLoc); + data = (void *)((Byte *)data + processedLoc); size -= processedLoc; } - while (size > 0); + while (size); return true; } // ---------- COutFile --------- -static inline DWORD GetCreationDisposition(bool createAlways) - { return createAlways? CREATE_ALWAYS: CREATE_NEW; } - bool COutFile::Open(CFSTR fileName, DWORD shareMode, DWORD creationDisposition, DWORD flagsAndAttributes) { return CFileBase::Create(fileName, GENERIC_WRITE, shareMode, creationDisposition, flagsAndAttributes); } -bool COutFile::Open(CFSTR fileName, DWORD creationDisposition) +bool COutFile::Open_Disposition(CFSTR fileName, DWORD creationDisposition) { return Open(fileName, FILE_SHARE_READ, creationDisposition, FILE_ATTRIBUTE_NORMAL); } -bool COutFile::Create(CFSTR fileName, bool createAlways) - { return Open(fileName, GetCreationDisposition(createAlways)); } - -bool COutFile::CreateAlways(CFSTR fileName, DWORD flagsAndAttributes) - { return Open(fileName, FILE_SHARE_READ, GetCreationDisposition(true), flagsAndAttributes); } +bool COutFile::Create_ALWAYS_with_Attribs(CFSTR fileName, DWORD flagsAndAttributes) + { return Open(fileName, FILE_SHARE_READ, CREATE_ALWAYS, flagsAndAttributes); } bool COutFile::SetTime(const FILETIME *cTime, const FILETIME *aTime, const FILETIME *mTime) throw() { return BOOLToBool(::SetFileTime(_handle, cTime, aTime, mTime)); } @@ -557,10 +566,10 @@ bool COutFile::Write(const void *data, UInt32 size, UInt32 &processedSize) throw return false; if (processedLoc == 0) return true; - data = (const void *)((const unsigned char *)data + processedLoc); + data = (const void *)((const Byte *)data + processedLoc); size -= processedLoc; } - while (size != 0); + while (size); return true; } @@ -574,10 +583,10 @@ bool COutFile::WriteFull(const void *data, size_t size) throw() return false; if (processedLoc == 0) return (size == 0); - data = (const void *)((const unsigned char *)data + processedLoc); + data = (const void *)((const Byte *)data + processedLoc); size -= processedLoc; } - while (size != 0); + while (size); return true; } @@ -786,11 +795,11 @@ bool CInFile::ReadFull(void *data, size_t size, size_t &processed) throw() return false; if (res == 0) break; - data = (void *)((unsigned char *)data + (size_t)res); - size -= (size_t)res; + data = (void *)((Byte *)data + (size_t)res); processed += (size_t)res; + size -= (size_t)res; } - while (size > 0); + while (size); return true; } @@ -798,23 +807,63 @@ bool CInFile::ReadFull(void *data, size_t size, size_t &processed) throw() ///////////////////////// // COutFile -bool COutFile::Create(const char *name, bool createAlways) +bool COutFile::OpenBinary_forWrite_oflag(const char *name, int oflag) { Path = name; // change it : set it only if open is success. - if (createAlways) - { - Close(); - _handle = ::creat(name, mode_for_Create); - return _handle != -1; - } - return OpenBinary(name, O_CREAT | O_EXCL | O_WRONLY, mode_for_Create); + return OpenBinary(name, oflag, mode_for_Create); } -bool COutFile::Open(const char *name, DWORD creationDisposition) + +/* + windows exist non-exist posix + CREATE_NEW Fail Create O_CREAT | O_EXCL + CREATE_ALWAYS Trunc Create O_CREAT | O_TRUNC + OPEN_ALWAYS Open Create O_CREAT + OPEN_EXISTING Open Fail 0 + TRUNCATE_EXISTING Trunc Fail O_TRUNC ??? + + // O_CREAT = If the file exists, this flag has no effect except as noted under O_EXCL below. + // If O_CREAT and O_EXCL are set, open() shall fail if the file exists. + // O_TRUNC : If the file exists and the file is successfully opened, its length shall be truncated to 0. +*/ +bool COutFile::Open_EXISTING(const char *name) + { return OpenBinary_forWrite_oflag(name, O_WRONLY); } +bool COutFile::Create_ALWAYS(const char *name) + { return OpenBinary_forWrite_oflag(name, O_WRONLY | O_CREAT | O_TRUNC); } +bool COutFile::Create_NEW(const char *name) + { return OpenBinary_forWrite_oflag(name, O_WRONLY | O_CREAT | O_EXCL); } +bool COutFile::Create_ALWAYS_or_Open_ALWAYS(const char *name, bool createAlways) +{ + return OpenBinary_forWrite_oflag(name, + createAlways ? + O_WRONLY | O_CREAT | O_TRUNC : + O_WRONLY | O_CREAT); +} +/* +bool COutFile::Create_ALWAYS_or_NEW(const char *name, bool createAlways) +{ + return OpenBinary_forWrite_oflag(name, + createAlways ? + O_WRONLY | O_CREAT | O_TRUNC : + O_WRONLY | O_CREAT | O_EXCL); +} +bool COutFile::Open_Disposition(const char *name, DWORD creationDisposition) { - UNUSED_VAR(creationDisposition) // FIXME - return Create(name, false); + int flag; + switch (creationDisposition) + { + case CREATE_NEW: flag = O_WRONLY | O_CREAT | O_EXCL; break; + case CREATE_ALWAYS: flag = O_WRONLY | O_CREAT | O_TRUNC; break; + case OPEN_ALWAYS: flag = O_WRONLY | O_CREAT; break; + case OPEN_EXISTING: flag = O_WRONLY; break; + case TRUNCATE_EXISTING: flag = O_WRONLY | O_TRUNC; break; + default: + SetLastError(EINVAL); + return false; + } + return OpenBinary_forWrite_oflag(name, flag); } +*/ ssize_t COutFile::write_part(const void *data, size_t size) throw() { @@ -833,11 +882,11 @@ ssize_t COutFile::write_full(const void *data, size_t size, size_t &processed) t return res; if (res == 0) break; - data = (const void *)((const unsigned char *)data + (size_t)res); - size -= (size_t)res; + data = (const void *)((const Byte *)data + (size_t)res); processed += (size_t)res; + size -= (size_t)res; } - while (size > 0); + while (size); return (ssize_t)processed; } diff --git a/CPP/Windows/FileIO.h b/CPP/Windows/FileIO.h index 03e061a..6ba40eb 100644 --- a/CPP/Windows/FileIO.h +++ b/CPP/Windows/FileIO.h @@ -141,13 +141,24 @@ class CFileBase MY_UNCOPYABLE public: bool PreserveATime; - #ifdef Z7_DEVICE_FILE +#if 0 + bool IsStdStream; + bool IsStdPipeStream; +#endif +#ifdef Z7_DEVICE_FILE bool IsDeviceFile; bool SizeDefined; UInt64 Size; // it can be larger than real available size - #endif +#endif - CFileBase(): _handle(INVALID_HANDLE_VALUE), PreserveATime(false) {} + CFileBase(): + _handle(INVALID_HANDLE_VALUE), + PreserveATime(false) +#if 0 + , IsStdStream(false), + , IsStdPipeStream(false) +#endif + {} ~CFileBase() { Close(); } HANDLE GetHandle() const { return _handle; } @@ -223,6 +234,20 @@ class CInFile: public CFileBase bool OpenShared(CFSTR fileName, bool shareForWrite); bool Open(CFSTR fileName); +#if 0 + bool AttachStdIn() + { + IsDeviceFile = false; + const HANDLE h = GetStdHandle(STD_INPUT_HANDLE); + if (h == INVALID_HANDLE_VALUE || !h) + return false; + IsStdStream = true; + IsStdPipeStream = true; + _handle = h; + return true; + } +#endif + #ifndef UNDER_CE bool Open_for_ReadAttributes(CFSTR fileName) @@ -263,11 +288,21 @@ class CInFile: public CFileBase class COutFile: public CFileBase { + bool Open_Disposition(CFSTR fileName, DWORD creationDisposition); public: bool Open(CFSTR fileName, DWORD shareMode, DWORD creationDisposition, DWORD flagsAndAttributes); - bool Open(CFSTR fileName, DWORD creationDisposition); - bool Create(CFSTR fileName, bool createAlways); - bool CreateAlways(CFSTR fileName, DWORD flagsAndAttributes); + bool Open_EXISTING(CFSTR fileName) + { return Open_Disposition(fileName, OPEN_EXISTING); } + bool Create_ALWAYS_or_Open_ALWAYS(CFSTR fileName, bool createAlways) + { return Open_Disposition(fileName, createAlways ? CREATE_ALWAYS : OPEN_ALWAYS); } + bool Create_ALWAYS_or_NEW(CFSTR fileName, bool createAlways) + { return Open_Disposition(fileName, createAlways ? CREATE_ALWAYS : CREATE_NEW); } + bool Create_ALWAYS(CFSTR fileName) + { return Open_Disposition(fileName, CREATE_ALWAYS); } + bool Create_NEW(CFSTR fileName) + { return Open_Disposition(fileName, CREATE_NEW); } + + bool Create_ALWAYS_with_Attribs(CFSTR fileName, DWORD flagsAndAttributes); bool SetTime(const CFiTime *cTime, const CFiTime *aTime, const CFiTime *mTime) throw(); bool SetMTime(const CFiTime *mTime) throw(); @@ -308,8 +343,15 @@ class CFileBase bool OpenBinary(const char *name, int flags, mode_t mode = 0666); public: bool PreserveATime; +#if 0 + bool IsStdStream; +#endif - CFileBase(): _handle(-1), PreserveATime(false) {} + CFileBase(): _handle(-1), PreserveATime(false) +#if 0 + , IsStdStream(false) +#endif + {} ~CFileBase() { Close(); } // void Detach() { _handle = -1; } bool Close(); @@ -331,6 +373,15 @@ class CInFile: public CFileBase public: bool Open(const char *name); bool OpenShared(const char *name, bool shareForWrite); +#if 0 + bool AttachStdIn() + { + _handle = GetStdHandle(STD_INPUT_HANDLE); + if (_handle == INVALID_HANDLE_VALUE || !_handle) + return false; + IsStdStream = true; + } +#endif ssize_t read_part(void *data, size_t size) throw(); // ssize_t read_full(void *data, size_t size, size_t &processed); bool ReadFull(void *data, size_t size, size_t &processedSize) throw(); @@ -347,6 +398,7 @@ class COutFile: public CFileBase AString Path; ssize_t write_part(const void *data, size_t size) throw(); + bool OpenBinary_forWrite_oflag(const char *name, int oflag); public: mode_t mode_for_Create; @@ -358,8 +410,14 @@ class COutFile: public CFileBase {} bool Close(); - bool Create(const char *name, bool createAlways); - bool Open(const char *name, DWORD creationDisposition); + + bool Open_EXISTING(CFSTR fileName); + bool Create_ALWAYS_or_Open_ALWAYS(CFSTR fileName, bool createAlways); + bool Create_ALWAYS(CFSTR fileName); + bool Create_NEW(CFSTR fileName); + // bool Create_ALWAYS_or_NEW(CFSTR fileName, bool createAlways); + // bool Open_Disposition(const char *name, DWORD creationDisposition); + ssize_t write_full(const void *data, size_t size, size_t &processed) throw(); bool WriteFull(const void *data, size_t size) throw() diff --git a/CPP/Windows/FileLink.cpp b/CPP/Windows/FileLink.cpp index 2b9fa1a..bb380ec 100644 --- a/CPP/Windows/FileLink.cpp +++ b/CPP/Windows/FileLink.cpp @@ -249,9 +249,13 @@ bool CReparseAttr::Parse(const Byte *p, size_t size) if (size < 8) return false; Tag = Get32(p); - UInt32 len = Get16(p + 4); - if (len + 8 != size) - // if (len + 8 > size) + if (Get16(p + 6) != 0) // padding + return false; + unsigned len = Get16(p + 4); + p += 8; + size -= 8; + if (len != size) + // if (len > size) return false; /* if ((type & kReparseFlags_Alias) == 0 || @@ -259,8 +263,6 @@ bool CReparseAttr::Parse(const Byte *p, size_t size) (type & 0xFFFF) != 3) */ - if (Get16(p + 6) != 0) // padding - return false; HeaderError = false; @@ -276,9 +278,6 @@ bool CReparseAttr::Parse(const Byte *p, size_t size) TagIsUnknown = false; - p += 8; - size -= 8; - if (Tag == Z7_WIN_IO_REPARSE_TAG_LX_SYMLINK) { if (len < 4) @@ -518,7 +517,7 @@ bool SetReparseData(CFSTR path, bool isDir, const void *data, DWORD size) { CreatePrefixDirOfFile(path); COutFile file; - if (!file.Create(path, CREATE_NEW)) + if (!file.Create_NEW(path)) return false; } } diff --git a/CPP/Windows/FileName.cpp b/CPP/Windows/FileName.cpp index c9c4f8b..c16b3d4 100644 --- a/CPP/Windows/FileName.cpp +++ b/CPP/Windows/FileName.cpp @@ -302,13 +302,13 @@ int FindAltStreamColon(CFSTR path) throw() static unsigned GetRootPrefixSize_Of_NetworkPath(CFSTR s) { // Network path: we look "server\path\" as root prefix - int pos = FindSepar(s); + const int pos = FindSepar(s); if (pos < 0) return 0; - int pos2 = FindSepar(s + (unsigned)pos + 1); + const int pos2 = FindSepar(s + (unsigned)pos + 1); if (pos2 < 0) return 0; - return pos + pos2 + 2; + return (unsigned)pos + (unsigned)pos2 + 2; } static unsigned GetRootPrefixSize_Of_SimplePath(CFSTR s) @@ -334,7 +334,7 @@ static unsigned GetRootPrefixSize_Of_SuperPath(CFSTR s) const int pos = FindSepar(s + kSuperPathPrefixSize); if (pos < 0) return 0; - return kSuperPathPrefixSize + pos + 1; + return kSuperPathPrefixSize + (unsigned)pos + 1; } unsigned GetRootPrefixSize(CFSTR s) throw() diff --git a/CPP/Windows/FileSystem.cpp b/CPP/Windows/FileSystem.cpp index 6a262d9..d11f02e 100644 --- a/CPP/Windows/FileSystem.cpp +++ b/CPP/Windows/FileSystem.cpp @@ -71,6 +71,12 @@ UINT MyGetDriveType(CFSTR pathName) } } +#if !defined(Z7_WIN32_WINNT_MIN) || Z7_WIN32_WINNT_MIN < 0x0400 +// GetDiskFreeSpaceEx requires Windows95-OSR2, NT4 +#define Z7_USE_DYN_GetDiskFreeSpaceEx +#endif + +#ifdef Z7_USE_DYN_GetDiskFreeSpaceEx typedef BOOL (WINAPI * Func_GetDiskFreeSpaceExA)( LPCSTR lpDirectoryName, // directory name PULARGE_INTEGER lpFreeBytesAvailable, // bytes available to caller @@ -84,6 +90,7 @@ typedef BOOL (WINAPI * Func_GetDiskFreeSpaceExW)( PULARGE_INTEGER lpTotalNumberOfBytes, // bytes on disk PULARGE_INTEGER lpTotalNumberOfFreeBytes // free bytes on disk ); +#endif bool MyGetDiskFreeSpace(CFSTR rootPath, UInt64 &clusterSize, UInt64 &totalSize, UInt64 &freeSize) { @@ -92,14 +99,22 @@ bool MyGetDiskFreeSpace(CFSTR rootPath, UInt64 &clusterSize, UInt64 &totalSize, #ifndef _UNICODE if (!g_IsNT) { +#ifdef Z7_USE_DYN_GetDiskFreeSpaceEx const Func_GetDiskFreeSpaceExA f = Z7_GET_PROC_ADDRESS( Func_GetDiskFreeSpaceExA, GetModuleHandle(TEXT("kernel32.dll")), "GetDiskFreeSpaceExA"); if (f) +#endif { ULARGE_INTEGER freeBytesToCaller2, totalSize2, freeSize2; - sizeIsDetected = BOOLToBool(f(fs2fas(rootPath), &freeBytesToCaller2, &totalSize2, &freeSize2)); + sizeIsDetected = BOOLToBool( +#ifdef Z7_USE_DYN_GetDiskFreeSpaceEx + f +#else + GetDiskFreeSpaceExA +#endif + (fs2fas(rootPath), &freeBytesToCaller2, &totalSize2, &freeSize2)); totalSize = totalSize2.QuadPart; freeSize = freeSize2.QuadPart; } @@ -109,14 +124,22 @@ bool MyGetDiskFreeSpace(CFSTR rootPath, UInt64 &clusterSize, UInt64 &totalSize, else #endif { +#ifdef Z7_USE_DYN_GetDiskFreeSpaceEx const Func_GetDiskFreeSpaceExW f = Z7_GET_PROC_ADDRESS( Func_GetDiskFreeSpaceExW, GetModuleHandle(TEXT("kernel32.dll")), "GetDiskFreeSpaceExW"); if (f) +#endif { ULARGE_INTEGER freeBytesToCaller2, totalSize2, freeSize2; - sizeIsDetected = BOOLToBool(f(fs2us(rootPath), &freeBytesToCaller2, &totalSize2, &freeSize2)); + sizeIsDetected = BOOLToBool( +#ifdef Z7_USE_DYN_GetDiskFreeSpaceEx + f +#else + GetDiskFreeSpaceExW +#endif + (fs2us(rootPath), &freeBytesToCaller2, &totalSize2, &freeSize2)); totalSize = totalSize2.QuadPart; freeSize = freeSize2.QuadPart; } diff --git a/CPP/Windows/MemoryLock.cpp b/CPP/Windows/MemoryLock.cpp index 0bd7504..bc2d7c7 100644 --- a/CPP/Windows/MemoryLock.cpp +++ b/CPP/Windows/MemoryLock.cpp @@ -37,6 +37,8 @@ bool EnablePrivilege(LPCTSTR privilegeName, bool enable) if (!hModule) return false; +Z7_DIAGNOSTIC_IGNORE_CAST_FUNCTION + GET_PROC_ADDR( OpenProcessToken, "OpenProcessToken") @@ -79,6 +81,7 @@ bool EnablePrivilege(LPCTSTR privilegeName, bool enable) } +Z7_DIAGNOSTIC_IGNORE_CAST_FUNCTION typedef void (WINAPI * Func_RtlGetVersion) (OSVERSIONINFOEXW *); diff --git a/CPP/Windows/ProcessUtils.cpp b/CPP/Windows/ProcessUtils.cpp index 607b4bb..f5e156c 100644 --- a/CPP/Windows/ProcessUtils.cpp +++ b/CPP/Windows/ProcessUtils.cpp @@ -17,7 +17,7 @@ static UString GetQuotedString(const UString &s) { UString s2 ('\"'); s2 += s; - s2 += '\"'; + s2.Add_Char('\"'); return s2; } #endif diff --git a/CPP/Windows/PropVariant.cpp b/CPP/Windows/PropVariant.cpp index 457b1dc..678f970 100644 --- a/CPP/Windows/PropVariant.cpp +++ b/CPP/Windows/PropVariant.cpp @@ -252,6 +252,7 @@ CPropVariant::~CPropVariant() throw() CASE_SIMPLE_VT_VALUES // vt = VT_EMPTY; // it's optional return; + default: break; } ::VariantClear((tagVARIANT *)this); } @@ -296,6 +297,7 @@ HRESULT CPropVariant::Copy(const PROPVARIANT* pSrc) throw() CASE_SIMPLE_VT_VALUES memmove((PROPVARIANT*)this, pSrc, sizeof(PROPVARIANT)); return S_OK; + default: break; } return ::VariantCopy((tagVARIANT *)this, (tagVARIANT *)const_cast(pSrc)); } diff --git a/CPP/Windows/PropVariantConv.cpp b/CPP/Windows/PropVariantConv.cpp index 5fb96a7..21a7b4e 100644 --- a/CPP/Windows/PropVariantConv.cpp +++ b/CPP/Windows/PropVariantConv.cpp @@ -9,12 +9,45 @@ #define UINT_TO_STR_2(c, val) { s[0] = (c); s[1] = (char)('0' + (val) / 10); s[2] = (char)('0' + (val) % 10); s += 3; } -bool ConvertUtcFileTimeToString2(const FILETIME &utc, unsigned ns100, char *s, int level) throw() +static const unsigned k_TimeStringBufferSize = 64; + +bool g_Timestamp_Show_UTC; +#if 0 +bool g_Timestamp_Show_DisableZ; +bool g_Timestamp_Show_TDelimeter; +bool g_Timestamp_Show_ZoneOffset; +#endif + +Z7_NO_INLINE +bool ConvertUtcFileTimeToString2(const FILETIME &utc, unsigned ns100, char *s, int level, unsigned flags) throw() { *s = 0; FILETIME ft; - if (!FileTimeToLocalFileTime(&utc, &ft)) - return false; + +#if 0 + Int64 bias64 = 0; +#endif + + const bool show_utc = + (flags & kTimestampPrintFlags_Force_UTC) ? true : + (flags & kTimestampPrintFlags_Force_LOCAL) ? false : + g_Timestamp_Show_UTC; + + if (show_utc) + ft = utc; + else + { + if (!FileTimeToLocalFileTime(&utc, &ft)) + return false; +#if 0 + if (g_Timestamp_Show_ZoneOffset) + { + const UInt64 utc64 = (((UInt64)utc.dwHighDateTime) << 32) + utc.dwLowDateTime; + const UInt64 loc64 = (((UInt64) ft.dwHighDateTime) << 32) + ft.dwLowDateTime; + bias64 = (Int64)utc64 - (Int64)loc64; + } +#endif + } SYSTEMTIME st; if (!BOOLToBool(FileTimeToSystemTime(&ft, &st))) @@ -41,7 +74,12 @@ bool ConvertUtcFileTimeToString2(const FILETIME &utc, unsigned ns100, char *s, i if (level > kTimestampPrintLevel_DAY) { - UINT_TO_STR_2(' ', st.wHour) + const char setChar = +#if 0 + g_Timestamp_Show_TDelimeter ? 'T' : // ISO 8601 +#endif + ' '; + UINT_TO_STR_2(setChar, st.wHour) UINT_TO_STR_2(':', st.wMinute) if (level >= kTimestampPrintLevel_SEC) @@ -84,6 +122,52 @@ bool ConvertUtcFileTimeToString2(const FILETIME &utc, unsigned ns100, char *s, i } } + if (show_utc) + { + if ((flags & kTimestampPrintFlags_DisableZ) == 0 +#if 0 + && !g_Timestamp_Show_DisableZ +#endif + ) + *s++ = 'Z'; + } +#if 0 + else if (g_Timestamp_Show_ZoneOffset) + { +#if 1 + { + char c; + if (bias64 < 0) + { + bias64 = -bias64; + c = '+'; + } + else + c = '-'; + UInt32 bias = (UInt32)((UInt64)bias64 / (10000000 * 60)); +#else + TIME_ZONE_INFORMATION zi; + const DWORD dw = GetTimeZoneInformation(&zi); + if (dw <= TIME_ZONE_ID_DAYLIGHT) // == 2 + { + // UTC = LOCAL + Bias + Int32 bias = zi.Bias; + char c; + if (bias < 0) + { + bias = -bias; + c = '+'; + } + else + c = '-'; +#endif + const UInt32 hours = (UInt32)bias / 60; + const UInt32 mins = (UInt32)bias % 60; + UINT_TO_STR_2(c, hours) + UINT_TO_STR_2(':', mins) + } + } +#endif *s = 0; return true; } @@ -96,11 +180,11 @@ bool ConvertUtcFileTimeToString(const FILETIME &utc, char *s, int level) throw() bool ConvertUtcFileTimeToString2(const FILETIME &ft, unsigned ns100, wchar_t *dest, int level) throw() { - char s[32]; - bool res = ConvertUtcFileTimeToString2(ft, ns100, s, level); + char s[k_TimeStringBufferSize]; + const bool res = ConvertUtcFileTimeToString2(ft, ns100, s, level); for (unsigned i = 0;; i++) { - Byte c = (Byte)s[i]; + const Byte c = (Byte)s[i]; dest[i] = c; if (c == 0) break; @@ -110,11 +194,11 @@ bool ConvertUtcFileTimeToString2(const FILETIME &ft, unsigned ns100, wchar_t *de bool ConvertUtcFileTimeToString(const FILETIME &ft, wchar_t *dest, int level) throw() { - char s[32]; - bool res = ConvertUtcFileTimeToString(ft, s, level); + char s[k_TimeStringBufferSize]; + const bool res = ConvertUtcFileTimeToString(ft, s, level); for (unsigned i = 0;; i++) { - Byte c = (Byte)s[i]; + const Byte c = (Byte)s[i]; dest[i] = c; if (c == 0) break; diff --git a/CPP/Windows/PropVariantConv.h b/CPP/Windows/PropVariantConv.h index ec5223b..8782507 100644 --- a/CPP/Windows/PropVariantConv.h +++ b/CPP/Windows/PropVariantConv.h @@ -7,6 +7,8 @@ // provide at least 32 bytes for buffer including zero-end +extern bool g_Timestamp_Show_UTC; + #define kTimestampPrintLevel_DAY -3 // #define kTimestampPrintLevel_HOUR -2 #define kTimestampPrintLevel_MIN -1 @@ -14,9 +16,14 @@ #define kTimestampPrintLevel_NTFS 7 #define kTimestampPrintLevel_NS 9 + +#define kTimestampPrintFlags_Force_UTC (1 << 0) +#define kTimestampPrintFlags_Force_LOCAL (1 << 1) +#define kTimestampPrintFlags_DisableZ (1 << 4) + bool ConvertUtcFileTimeToString(const FILETIME &ft, char *s, int level = kTimestampPrintLevel_SEC) throw(); bool ConvertUtcFileTimeToString(const FILETIME &ft, wchar_t *s, int level = kTimestampPrintLevel_SEC) throw(); -bool ConvertUtcFileTimeToString2(const FILETIME &ft, unsigned ns100, char *s, int level = kTimestampPrintLevel_SEC) throw(); +bool ConvertUtcFileTimeToString2(const FILETIME &ft, unsigned ns100, char *s, int level = kTimestampPrintLevel_SEC, unsigned flags = 0) throw(); bool ConvertUtcFileTimeToString2(const FILETIME &ft, unsigned ns100, wchar_t *s, int level = kTimestampPrintLevel_SEC) throw(); // provide at least 32 bytes for buffer including zero-end diff --git a/CPP/Windows/Registry.cpp b/CPP/Windows/Registry.cpp index b20157d..c8b1709 100644 --- a/CPP/Windows/Registry.cpp +++ b/CPP/Windows/Registry.cpp @@ -254,8 +254,8 @@ LONG CKey::QueryValue(LPCTSTR name, CSysString &value) { value.Empty(); DWORD type = 0; - UInt32 curSize = 0; - LONG res = RegQueryValueEx(_object, name, NULL, &type, NULL, (DWORD *)&curSize); + DWORD curSize = 0; + LONG res = RegQueryValueEx(_object, name, NULL, &type, NULL, &curSize); if (res != ERROR_SUCCESS && res != ERROR_MORE_DATA) return res; UInt32 curSize2 = curSize; @@ -281,13 +281,11 @@ LONG CKey::QueryValue(LPCWSTR name, UString &value) { value.Empty(); DWORD type = 0; - UInt32 curSize = 0; - + DWORD curSize = 0; LONG res; - if (g_IsNT) { - res = RegQueryValueExW(_object, name, NULL, &type, NULL, (DWORD *)&curSize); + res = RegQueryValueExW(_object, name, NULL, &type, NULL, &curSize); if (res != ERROR_SUCCESS && res != ERROR_MORE_DATA) return res; UInt32 curSize2 = curSize; @@ -302,7 +300,6 @@ LONG CKey::QueryValue(LPCWSTR name, UString &value) res = QueryValue(name == 0 ? 0 : (LPCSTR)GetSystemString(name), vTemp); value = GetUnicodeString(vTemp); } - return res; } @@ -377,7 +374,7 @@ LONG CKey::GetValue_Strings(LPCTSTR valueName, UStringVector &strings) strings.Clear(); CByteBuffer buffer; UInt32 dataSize = 0; - LONG res = QueryValue(valueName, buffer, dataSize); + const LONG res = QueryValue(valueName, buffer, dataSize); if (res != ERROR_SUCCESS) return res; if (dataSize > buffer.Size()) @@ -386,7 +383,7 @@ LONG CKey::GetValue_Strings(LPCTSTR valueName, UStringVector &strings) return E_FAIL; const wchar_t *data = (const wchar_t *)(const void *)(const Byte *)buffer; - size_t numChars = dataSize / sizeof(wchar_t); + const size_t numChars = dataSize / sizeof(wchar_t); size_t prev = 0; UString s; diff --git a/CPP/Windows/SecurityUtils.h b/CPP/Windows/SecurityUtils.h index 4ef3939..7219f06 100644 --- a/CPP/Windows/SecurityUtils.h +++ b/CPP/Windows/SecurityUtils.h @@ -18,6 +18,8 @@ typedef NTSTATUS (NTAPI *Func_LsaAddAccountRights)(LSA_HANDLE PolicyHandle, #define MY_STATUS_NOT_IMPLEMENTED ((NTSTATUS)0xC0000002L) } +Z7_DIAGNOSTIC_IGNORE_CAST_FUNCTION + #define POLICY_FUNC_CALL(fff, str) \ if (hModule == NULL) return MY_STATUS_NOT_IMPLEMENTED; \ const Func_ ## fff v = Z7_GET_PROC_ADDRESS(Func_ ## fff, hModule, str); \ diff --git a/CPP/Windows/Shell.cpp b/CPP/Windows/Shell.cpp index b2a3489..01ceb22 100644 --- a/CPP/Windows/Shell.cpp +++ b/CPP/Windows/Shell.cpp @@ -162,7 +162,7 @@ static HRESULT ReadAnsiStrings(const char *p, size_t size, UStringVector &names) name.Empty(); } else - name += c; + name.Add_Char(c); } return E_INVALIDARG; } @@ -543,8 +543,18 @@ void CDrop::QueryFileNames(UStringVector &fileNames) typedef int Z7_WIN_GPFIDL_FLAGS; extern "C" { -typedef BOOL (WINAPI * Func_SHGetPathFromIDListW)(LPCITEMIDLIST pidl, LPWSTR pszPath); -typedef BOOL (WINAPI * Func_SHGetPathFromIDListEx)(LPCITEMIDLIST pidl, PWSTR pszPath, DWORD cchPath, Z7_WIN_GPFIDL_FLAGS uOpts); +#ifndef _UNICODE +typedef BOOL (WINAPI * Func_SHGetPathFromIDListW)(LPCITEMIDLIST pidl, LPWSTR pszPath); // nt4 +#endif + +#if !defined(Z7_WIN32_WINNT_MIN) || Z7_WIN32_WINNT_MIN < 0x0600 // Vista +#define Z7_USE_DYN_SHGetPathFromIDListEx +#endif + +#ifdef Z7_USE_DYN_SHGetPathFromIDListEx +Z7_DIAGNOSTIC_IGNORE_CAST_FUNCTION +typedef BOOL (WINAPI * Func_SHGetPathFromIDListEx)(LPCITEMIDLIST pidl, PWSTR pszPath, DWORD cchPath, Z7_WIN_GPFIDL_FLAGS uOpts); // vista +#endif } #ifndef _UNICODE @@ -584,18 +594,26 @@ bool GetPathFromIDList(LPCITEMIDLIST itemIDList, UString &path) /* for long path we need SHGetPathFromIDListEx(). win10: SHGetPathFromIDListEx() for long path returns path with with super path prefix "\\\\?\\". */ +#ifdef Z7_USE_DYN_SHGetPathFromIDListEx const Func_SHGetPathFromIDListEx func_SHGetPathFromIDListEx = Z7_GET_PROC_ADDRESS( Func_SHGetPathFromIDListEx, ::GetModuleHandleW(L"shell32.dll"), "SHGetPathFromIDListEx"); if (func_SHGetPathFromIDListEx) +#endif { ODS("==== GetPathFromIDList() (SHGetPathFromIDListEx)") do { len *= 4; - result = BOOLToBool(func_SHGetPathFromIDListEx(itemIDList, path.GetBuf(len), len, 0)); + result = BOOLToBool( +#ifdef Z7_USE_DYN_SHGetPathFromIDListEx + func_SHGetPathFromIDListEx +#else + SHGetPathFromIDListEx +#endif + (itemIDList, path.GetBuf(len), len, 0)); if (result) break; } diff --git a/CPP/Windows/Shell.h b/CPP/Windows/Shell.h index b4cdb30..9e58e25 100644 --- a/CPP/Windows/Shell.h +++ b/CPP/Windows/Shell.h @@ -3,7 +3,6 @@ #ifndef ZIP7_WINDOWS_SHELL_H #define ZIP7_WINDOWS_SHELL_H -#include "../Common/Common.h" #include "../Common/MyWindows.h" #if defined(__MINGW32__) || defined(__MINGW64__) #include diff --git a/CPP/Windows/StdAfx.h b/CPP/Windows/StdAfx.h index bd5084f..4441a5f 100644 --- a/CPP/Windows/StdAfx.h +++ b/CPP/Windows/StdAfx.h @@ -6,7 +6,6 @@ #if defined(_MSC_VER) && _MSC_VER >= 1800 #pragma warning(disable : 4464) // relative include path contains '..' #endif - #include "../Common/Common.h" #endif diff --git a/CPP/Windows/Synchronization.h b/CPP/Windows/Synchronization.h index afd03d2..1df84d3 100644 --- a/CPP/Windows/Synchronization.h +++ b/CPP/Windows/Synchronization.h @@ -234,6 +234,11 @@ class CSynchro MY_UNCOPYABLE } WRes Enter() { +#if defined(Z7_LLVM_CLANG_VERSION) && (__clang_major__ == 13) \ + && defined(__FreeBSD__) + #pragma GCC diagnostic ignored "-Wthread-safety-negative" + #pragma GCC diagnostic ignored "-Wthread-safety-analysis" +#endif return ::pthread_mutex_lock(&_mutex); } WRes Leave() diff --git a/CPP/Windows/System.cpp b/CPP/Windows/System.cpp index dbe287a..03c8988 100644 --- a/CPP/Windows/System.cpp +++ b/CPP/Windows/System.cpp @@ -5,7 +5,8 @@ #ifndef _WIN32 #include #include -#ifdef __APPLE__ +#if defined(__APPLE__) || defined(__DragonFly__) || \ + defined(BSD) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) #include #else #include @@ -28,7 +29,10 @@ UInt32 CountAffinity(DWORD_PTR mask) { UInt32 num = 0; for (unsigned i = 0; i < sizeof(mask) * 8; i++) - num += (UInt32)((mask >> i) & 1); + { + num += (UInt32)(mask & 1); + mask >>= 1; + } return num; } @@ -114,7 +118,7 @@ UInt32 GetNumberOfProcessors() #if !defined(_WIN64) && \ (defined(__MINGW32_VERSION) || defined(Z7_OLD_WIN_SDK)) -typedef struct _MY_MEMORYSTATUSEX { +typedef struct { DWORD dwLength; DWORD dwMemoryLoad; DWORDLONG ullTotalPhys; @@ -187,26 +191,58 @@ bool GetRamSize(UInt64 &size) { size = (UInt64)(sizeof(size_t)) << 29; - #ifdef __APPLE__ +#if defined(__APPLE__) || defined(__DragonFly__) || \ + defined(BSD) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) + + uint64_t val = 0; + int mib[2]; + mib[0] = CTL_HW; #ifdef HW_MEMSIZE - uint64_t val = 0; // support 2Gb+ RAM - int mib[2] = { CTL_HW, HW_MEMSIZE }; + mib[1] = HW_MEMSIZE; + // printf("\n sysctl HW_MEMSIZE"); #elif defined(HW_PHYSMEM64) - uint64_t val = 0; // support 2Gb+ RAM - int mib[2] = { CTL_HW, HW_PHYSMEM64 }; + mib[1] = HW_PHYSMEM64; + // printf("\n sysctl HW_PHYSMEM64"); #else - unsigned int val = 0; // For old system - int mib[2] = { CTL_HW, HW_PHYSMEM }; - #endif // HW_MEMSIZE - size_t size_sys = sizeof(val); + mib[1] = HW_PHYSMEM; + // printf("\n sysctl HW_PHYSMEM"); + #endif - sysctl(mib, 2, &val, &size_sys, NULL, 0); - if (val) - size = val; + size_t size_sys = sizeof(val); + int res = sysctl(mib, 2, &val, &size_sys, NULL, 0); + // printf("\n sysctl res=%d val=%llx size_sys = %d, %d\n", res, (long long int)val, (int)size_sys, errno); + // we use strict check (size_sys == sizeof(val)) for returned value + // because big-endian encoding is possible: + if (res == 0 && size_sys == sizeof(val) && val) + size = val; + else + { + uint32_t val32 = 0; + size_sys = sizeof(val32); + res = sysctl(mib, 2, &val32, &size_sys, NULL, 0); + // printf("\n sysctl res=%d val=%llx size_sys = %d, %d\n", res, (long long int)val32, (int)size_sys, errno); + if (res == 0 && size_sys == sizeof(val32) && val32) + size = val32; + } #elif defined(_AIX) - // fixme + #if defined(_SC_AIX_REALMEM) // AIX + size = (UInt64)sysconf(_SC_AIX_REALMEM) * 1024; + #endif + #elif 0 || defined(__sun) + #if defined(_SC_PHYS_PAGES) && defined(_SC_PAGESIZE) + // FreeBSD, Linux, OpenBSD, and Solaris. + { + const long phys_pages = sysconf(_SC_PHYS_PAGES); + const long page_size = sysconf(_SC_PAGESIZE); + // #pragma message("GetRamSize : sysconf(_SC_PHYS_PAGES) * sysconf(_SC_PAGESIZE)") + // printf("\n_SC_PHYS_PAGES (hex) = %lx", (unsigned long)phys_pages); + // printf("\n_SC_PAGESIZE = %lu\n", (unsigned long)page_size); + if (phys_pages != -1 && page_size != -1) + size = (UInt64)(Int64)phys_pages * (UInt64)(Int64)page_size; + } + #endif #elif defined(__gnu_hurd__) // fixme #elif defined(__FreeBSD_kernel__) && defined(__GLIBC__) @@ -218,10 +254,6 @@ bool GetRamSize(UInt64 &size) if (::sysinfo(&info) != 0) return false; size = (UInt64)info.mem_unit * info.totalram; - const UInt64 kLimit = (UInt64)1 << (sizeof(size_t) * 8 - 1); - if (size > kLimit) - size = kLimit; - /* printf("\n mem_unit = %lld", (UInt64)info.mem_unit); printf("\n totalram = %lld", (UInt64)info.totalram); @@ -230,6 +262,10 @@ bool GetRamSize(UInt64 &size) #endif + const UInt64 kLimit = (UInt64)1 << (sizeof(size_t) * 8 - 1); + if (size > kLimit) + size = kLimit; + return true; } @@ -269,7 +305,7 @@ unsigned Get_File_OPEN_MAX_Reduced_for_3_tasks() numFiles_OPEN_MAX = 1; numFiles_OPEN_MAX /= 3; // we suppose that we have up to 3 tasks in total for multiple file processing numFiles_OPEN_MAX = MyMax(numFiles_OPEN_MAX, (unsigned long)3); - unsigned n = (UInt32)(UInt32)-1; + unsigned n = (unsigned)(int)-1; if (n > numFiles_OPEN_MAX) n = (unsigned)numFiles_OPEN_MAX; return n; diff --git a/CPP/Windows/System.h b/CPP/Windows/System.h index 0650007..b17111c 100644 --- a/CPP/Windows/System.h +++ b/CPP/Windows/System.h @@ -105,7 +105,7 @@ struct CProcessAffinity } void CpuZero() { } - void CpuSet(unsigned cpuIndex) { UNUSED_VAR(cpuIndex); } + void CpuSet(unsigned /* cpuIndex */) { /* UNUSED_VAR(cpuIndex) */ } int IsCpuSet(unsigned cpuIndex) const { return (cpuIndex < numSysThreads) ? 1 : 0; } BOOL SetProcAffinity() const diff --git a/CPP/Windows/SystemInfo.cpp b/CPP/Windows/SystemInfo.cpp index 6bafc80..d23e84b 100644 --- a/CPP/Windows/SystemInfo.cpp +++ b/CPP/Windows/SystemInfo.cpp @@ -16,9 +16,28 @@ #include #ifdef __APPLE__ #include + #elif !defined(_AIX) +#if defined(__GLIBC__) && (__GLIBC__ * 100 + __GLIBC_MINOR__ >= 216) + #define Z7_GETAUXV_AVAILABLE +#else +// #pragma message("=== is not NEW GLIBC === ") + #if defined __has_include + #if __has_include () +// #pragma message("=== sys/auxv.h is avail=== ") + #define Z7_GETAUXV_AVAILABLE + #endif + #endif +#endif + +#ifdef Z7_GETAUXV_AVAILABLE +// #if defined __has_include +// #if __has_include () #include +#define USE_HWCAP +// #endif +// #endif // #undef AT_HWCAP // to debug // #undef AT_HWCAP2 // to debug @@ -36,11 +55,39 @@ #endif */ +#ifdef USE_HWCAP + +#if defined(__FreeBSD__) + +// #if (__FreeBSD__ >= 13) // (FreeBSD 12.01 is required for elf_aux_info() ???) +static unsigned long MY_getauxval(int aux) +{ + unsigned long val; + if (elf_aux_info(aux, &val, sizeof(val))) + return 0; + return val; +} + +#else // ! __FreeBSD__ + #ifdef MY_CPU_ARM_OR_ARM64 + #if defined __has_include + #if __has_include () #include + #endif + #endif #endif + +#if defined(AT_HWCAP) || defined(AT_HWCAP2) +#define MY_getauxval getauxval #endif +#endif // ! __FreeBSD__ +#endif // USE_HWCAP +#endif // Z7_GETAUXV_AVAILABLE + +#endif // !defined(_AIX) + #ifdef __linux__ #include "../Windows/FileIO.h" #endif @@ -70,13 +117,13 @@ static bool ReadFile_to_Buffer(CFSTR fileName, CByteBuffer &buf) return false; */ size_t size = 0; - size_t addSize = ((size_t)1 << 12); + size_t addSize = (size_t)1 << 12; for (;;) { // printf("\nsize = %d\n", (unsigned)size); buf.ChangeSize_KeepData(size + addSize, size); size_t processed; - if (!file.ReadFull(buf + size, addSize, processed)) + if (!file.ReadFull(buf.NonConstData() + size, addSize, processed)) return false; if (processed == 0) { @@ -107,19 +154,19 @@ static void PrintCpuChars(AString &s, UInt32 v) { for (unsigned j = 0; j < 4; j++) { - Byte b = (Byte)(v & 0xFF); + const Byte b = (Byte)(v & 0xFF); v >>= 8; if (b == 0) break; if (b >= 0x20 && b <= 0x7f) - s += (char)b; + s.Add_Char((char)b); else { - s += '['; + s.Add_Char('['); char temp[16]; ConvertUInt32ToHex(b, temp); s += temp; - s += ']'; + s.Add_Char(']'); } } } @@ -137,7 +184,7 @@ static void x86cpuid_to_String(AString &s) { for (unsigned i = 0; i < 3; i++) { - z7_x86_cpuid(a, 0x80000002 + i); + z7_x86_cpuid(a, (UInt32)(0x80000002 + i)); for (unsigned j = 0; j < 4; j++) PrintCpuChars(s, a[j]); } @@ -353,8 +400,8 @@ void PrintSize_KMGT_Or_Hex(AString &s, UInt64 v) } s.Add_UInt64(v); if (c) - s += c; - s += 'B'; + s.Add_Char(c); + s.Add_Char('B'); } // #endif // #endif @@ -380,7 +427,7 @@ static void SysInfo_To_String(AString &s, const SYSTEM_INFO &si) s += " act:"; PrintHex(s, si.dwActiveProcessorMask); } - s += " cpus:"; + s += " threads:"; s.Add_UInt32(si.dwNumberOfProcessors); if (si.dwPageSize != 1 << 12) { @@ -400,7 +447,7 @@ static void SysInfo_To_String(AString &s, const SYSTEM_INFO &si) if (minAdd != kReserveSize) { PrintSize_KMGT_Or_Hex(s, minAdd); - s += "-"; + s.Add_Minus(); } else { @@ -470,11 +517,10 @@ static void AddBracedString(AString &dest, AString &src) { if (!src.IsEmpty()) { - AString s; - s += '('; - s += src; - s += ')'; - dest.Add_OptSpaced(s); + dest.Add_Space_if_NotEmpty(); + dest.Add_Char('('); + dest += src; + dest.Add_Char(')'); } } @@ -527,6 +573,13 @@ void CCpuName::Fill() { Add_sysctlbyname_to_String("machdep.cpu.brand_string", s); } + #elif defined(MY_CPU_E2K) && defined(Z7_MCST_LCC_VERSION) && (Z7_MCST_LCC_VERSION >= 12323) + { + s += "mcst "; + s += __builtin_cpu_name(); + s.Add_Space(); + s += __builtin_cpu_arch(); + } #endif @@ -592,11 +645,11 @@ void CCpuName::Fill() const CByteBuffer &buf = bufs[i]; if (buf.Size() == 8) { - UInt32 high = GetUi32(buf); + const UInt32 high = GetUi32(buf); if (high != 0) { PrintHex(Microcode, high); - Microcode += "."; + Microcode.Add_Dot(); } PrintHex(Microcode, GetUi32(buf + 4)); } @@ -622,7 +675,7 @@ void AddCpuFeatures(AString &s) UInt64 flags = 0; for (unsigned i = 0; i < kNumFeatures; i++) { - if (IsProcessorFeaturePresent(i)) + if (IsProcessorFeaturePresent((DWORD)i)) { flags += (UInt64)1 << i; // s.Add_Space_if_NotEmpty(); @@ -682,7 +735,7 @@ void AddCpuFeatures(AString &s) #ifdef AT_HWCAP s.Add_OptSpaced("hwcap:"); { - unsigned long h = getauxval(AT_HWCAP); + unsigned long h = MY_getauxval(AT_HWCAP); PrintHex(s, h); #ifdef MY_CPU_ARM64 if (h & HWCAP_CRC32) s += ":CRC32"; @@ -698,7 +751,7 @@ void AddCpuFeatures(AString &s) #ifdef AT_HWCAP2 { - unsigned long h = getauxval(AT_HWCAP2); + unsigned long h = MY_getauxval(AT_HWCAP2); #ifndef MY_CPU_ARM if (h != 0) #endif @@ -722,6 +775,8 @@ void AddCpuFeatures(AString &s) #ifdef _WIN32 #ifndef UNDER_CE +Z7_DIAGNOSTIC_IGNORE_CAST_FUNCTION + EXTERN_C_BEGIN typedef void (WINAPI * Func_RtlGetVersion) (OSVERSIONINFOEXW *); EXTERN_C_END @@ -952,24 +1007,13 @@ void GetVirtCpuid(AString &s) void GetCompiler(AString &s) { - #ifdef __VERSION__ - s += __VERSION__; - #endif - - #ifdef __GNUC__ - s += " GCC "; - s.Add_UInt32(__GNUC__); - s.Add_Dot(); - s.Add_UInt32(__GNUC_MINOR__); - s.Add_Dot(); - s.Add_UInt32(__GNUC_PATCHLEVEL__); - #endif - #ifdef __clang__ s += " CLANG "; s.Add_UInt32(__clang_major__); s.Add_Dot(); s.Add_UInt32(__clang_minor__); + s.Add_Dot(); + s.Add_UInt32(__clang_patchlevel__); #endif #ifdef __xlC__ @@ -985,12 +1029,67 @@ void GetCompiler(AString &s) #endif #endif + // #define __LCC__ 126 + // #define __LCC_MINOR__ 20 + // #define __MCST__ 1 + #ifdef __MCST__ + s += " MCST"; + #endif + #ifdef __LCC__ + s += " LCC "; + s.Add_UInt32(__LCC__ / 100); + s.Add_Dot(); + s.Add_UInt32(__LCC__ % 100 / 10); + s.Add_UInt32(__LCC__ % 10); + #ifdef __LCC_MINOR__ + s.Add_Dot(); + s.Add_UInt32(__LCC_MINOR__ / 10); + s.Add_UInt32(__LCC_MINOR__ % 10); + #endif + #endif + + // #define __EDG_VERSION__ 602 + #ifdef __EDG_VERSION__ + s += " EDG "; + s.Add_UInt32(__EDG_VERSION__ / 100); + s.Add_Dot(); + s.Add_UInt32(__EDG_VERSION__ % 100 / 10); + s.Add_UInt32(__EDG_VERSION__ % 10); + #endif + + #ifdef __VERSION__ + s.Add_Space(); + s += "ver:"; + s += __VERSION__; + #endif + + #ifdef __GNUC__ + s += " GCC "; + s.Add_UInt32(__GNUC__); + s.Add_Dot(); + s.Add_UInt32(__GNUC_MINOR__); + s.Add_Dot(); + s.Add_UInt32(__GNUC_PATCHLEVEL__); + #endif + + #ifdef _MSC_VER s += " MSC "; s.Add_UInt32(_MSC_VER); + #ifdef _MSC_FULL_VER + s.Add_Dot(); + s.Add_UInt32(_MSC_FULL_VER); + #endif + #endif - #if defined(__AVX2__) + #if defined(__AVX512F__) + #if defined(__AVX512VL__) + #define MY_CPU_COMPILE_ISA "AVX512VL" + #else + #define MY_CPU_COMPILE_ISA "AVX512F" + #endif + #elif defined(__AVX2__) #define MY_CPU_COMPILE_ISA "AVX2" #elif defined(__AVX__) #define MY_CPU_COMPILE_ISA "AVX" @@ -1014,9 +1113,61 @@ void GetCompiler(AString &s) #define MY_CPU_COMPILE_ISA "IA32" #endif + AString s2; #ifdef MY_CPU_COMPILE_ISA - s += ':'; - s.Add_OptSpaced(MY_CPU_COMPILE_ISA); + s2.Add_OptSpaced(MY_CPU_COMPILE_ISA); + #endif + +#ifndef MY_CPU_ARM64 + #ifdef __ARM_FP + s2.Add_OptSpaced("FP"); + #endif + #ifdef __ARM_NEON + s2.Add_OptSpaced("NEON"); #endif + #ifdef __NEON__ + s2.Add_OptSpaced("__NEON__"); + #endif + #ifdef __ARM_FEATURE_SIMD32 + s2.Add_OptSpaced("SIMD32"); + #endif +#endif + + #ifdef __ARM_FEATURE_CRYPTO + s2.Add_OptSpaced("CRYPTO"); + #endif + + #ifdef __ARM_FEATURE_SHA2 + s2.Add_OptSpaced("SHA2"); + #endif + + #ifdef __ARM_FEATURE_AES + s2.Add_OptSpaced("AES"); + #endif + + #ifdef __ARM_FEATURE_CRC32 + s2.Add_OptSpaced("CRC32"); + #endif + + #ifdef __ARM_FEATURE_UNALIGNED + s2.Add_OptSpaced("UNALIGNED"); + #endif + + + #ifdef MY_CPU_BE + s2.Add_OptSpaced("BE"); + #endif + + #if defined(MY_CPU_LE_UNALIGN) \ + && !defined(MY_CPU_X86_OR_AMD64) \ + && !defined(MY_CPU_ARM64) + s2.Add_OptSpaced("LE-unaligned"); + #endif + + if (!s2.IsEmpty()) + { + s.Add_OptSpaced(": "); + s += s2; + } } diff --git a/CPP/Windows/TimeUtils.cpp b/CPP/Windows/TimeUtils.cpp index e80ae13..bbd79ba 100644 --- a/CPP/Windows/TimeUtils.cpp +++ b/CPP/Windows/TimeUtils.cpp @@ -4,6 +4,7 @@ #ifndef _WIN32 #include +#include #endif #include "Defs.h" @@ -13,13 +14,13 @@ namespace NWindows { namespace NTime { static const UInt32 kNumTimeQuantumsInSecond = 10000000; -static const UInt32 kFileTimeStartYear = 1601; +static const unsigned kFileTimeStartYear = 1601; #if !defined(_WIN32) || defined(UNDER_CE) -static const UInt32 kDosTimeStartYear = 1980; +static const unsigned kDosTimeStartYear = 1980; #endif -static const UInt32 kUnixTimeStartYear = 1970; +static const unsigned kUnixTimeStartYear = 1970; static const UInt64 kUnixTimeOffset = - (UInt64)60 * 60 * 24 * (89 + 365 * (kUnixTimeStartYear - kFileTimeStartYear)); + (UInt64)60 * 60 * 24 * (89 + 365 * (UInt32)(kUnixTimeStartYear - kFileTimeStartYear)); static const UInt64 kNumSecondsInFileTime = (UInt64)(Int64)-1 / kNumTimeQuantumsInSecond; bool DosTime_To_FileTime(UInt32 dosTime, FILETIME &ft) throw() @@ -30,8 +31,14 @@ bool DosTime_To_FileTime(UInt32 dosTime, FILETIME &ft) throw() ft.dwLowDateTime = 0; ft.dwHighDateTime = 0; UInt64 res; - if (!GetSecondsSince1601(kDosTimeStartYear + (dosTime >> 25), (dosTime >> 21) & 0xF, (dosTime >> 16) & 0x1F, - (dosTime >> 11) & 0x1F, (dosTime >> 5) & 0x3F, (dosTime & 0x1F) * 2, res)) + if (!GetSecondsSince1601( + kDosTimeStartYear + (unsigned)(dosTime >> 25), + (unsigned)((dosTime >> 21) & 0xF), + (unsigned)((dosTime >> 16) & 0x1F), + (unsigned)((dosTime >> 11) & 0x1F), + (unsigned)((dosTime >> 5) & 0x3F), + (unsigned)((dosTime & 0x1F)) * 2, + res)) return false; res *= kNumTimeQuantumsInSecond; ft.dwLowDateTime = (UInt32)res; @@ -77,7 +84,7 @@ bool FileTime_To_DosTime(const FILETIME &ft, UInt32 &dosTime) throw() v = (UInt32)v64; - year = (unsigned)(kFileTimeStartYear + v / PERIOD_400 * 400); + year = kFileTimeStartYear + (unsigned)(v / PERIOD_400 * 400); v %= PERIOD_400; temp = (unsigned)(v / PERIOD_100); @@ -116,7 +123,13 @@ bool FileTime_To_DosTime(const FILETIME &ft, UInt32 &dosTime) throw() dosTime = kHighDosTime; if (year >= 128) return false; - dosTime = (year << 25) | (mon << 21) | (day << 16) | (hour << 11) | (min << 5) | (sec >> 1); + dosTime = + ((UInt32)year << 25) + | ((UInt32)mon << 21) + | ((UInt32)day << 16) + | ((UInt32)hour << 11) + | ((UInt32)min << 5) + | ((UInt32)sec >> 1); #endif return true; } @@ -226,20 +239,34 @@ bool GetSecondsSince1601(unsigned year, unsigned month, unsigned day, if (year < kFileTimeStartYear || year >= 10000 || month < 1 || month > 12 || day < 1 || day > 31 || hour > 23 || min > 59 || sec > 59) return false; - UInt32 numYears = year - kFileTimeStartYear; - UInt32 numDays = numYears * 365 + numYears / 4 - numYears / 100 + numYears / 400; + const unsigned numYears = year - kFileTimeStartYear; + UInt32 numDays = (UInt32)((UInt32)numYears * 365 + numYears / 4 - numYears / 100 + numYears / 400); Byte ms[] = { 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 }; if (year % 4 == 0 && (year % 100 != 0 || year % 400 == 0)) ms[1] = 29; month--; for (unsigned i = 0; i < month; i++) numDays += ms[i]; - numDays += day - 1; + numDays += (UInt32)(day - 1); resSeconds = ((UInt64)(numDays * 24 + hour) * 60 + min) * 60 + sec; return true; } +/* docs: TIME_UTC is not defined on many platforms: + glibc 2.15, macOS 10.13 + FreeBSD 11.0, NetBSD 7.1, OpenBSD 6.0, + Minix 3.1.8, AIX 7.1, HP-UX 11.31, IRIX 6.5, Solaris 11.3, + Cygwin 2.9, mingw, MSVC 14, Android 9.0. +*/ +#if defined(TIME_UTC) +#define ZIP7_USE_timespec_get +// #pragma message("ZIP7_USE_timespec_get") +#elif defined(CLOCK_REALTIME) +#define ZIP7_USE_clock_gettime +// #pragma message("ZIP7_USE_clock_gettime") +#endif + void GetCurUtc_FiTime(CFiTime &ft) throw() { #ifdef _WIN32 @@ -257,12 +284,33 @@ void GetCurUtc_FiTime(CFiTime &ft) throw() #else FiTime_Clear(ft); +#ifdef ZIP7_USE_timespec_get + timespec_get(&ft, TIME_UTC); +#elif defined ZIP7_USE_clock_gettime + +#if defined(_AIX) + { + timespec ts; + clock_gettime(CLOCK_REALTIME, &ts); + ft.tv_sec = ts.tv_sec; + ft.tv_nsec = ts.tv_nsec; + } +#else + clock_gettime(CLOCK_REALTIME, &ft); +#endif + +#else struct timeval now; if (gettimeofday(&now, NULL) == 0) { ft.tv_sec = now.tv_sec; - ft.tv_nsec = now.tv_usec * 1000; + // timeval::tv_usec can be 64-bit signed in some cases + // timespec::tv_nsec can be 32-bit signed in some cases + ft.tv_nsec = + (Int32) // to eliminate compiler conversion error + (now.tv_usec * 1000); } +#endif #endif } @@ -271,12 +319,26 @@ void GetCurUtc_FiTime(CFiTime &ft) throw() void GetCurUtcFileTime(FILETIME &ft) throw() { UInt64 v = 0; +#if defined(ZIP7_USE_timespec_get) || \ + defined(ZIP7_USE_clock_gettime) + timespec ts; +#if defined(ZIP7_USE_timespec_get) + if (timespec_get(&ts, TIME_UTC)) +#else + if (clock_gettime(CLOCK_REALTIME, &ts) == 0) +#endif + { + v = ((UInt64)ts.tv_sec + kUnixTimeOffset) * + kNumTimeQuantumsInSecond + (UInt64)ts.tv_nsec / 100; + } +#else struct timeval now; if (gettimeofday(&now, NULL) == 0) { v = ((UInt64)now.tv_sec + kUnixTimeOffset) * kNumTimeQuantumsInSecond + (UInt64)now.tv_usec * 10; } +#endif ft.dwLowDateTime = (DWORD)v; ft.dwHighDateTime = (DWORD)(v >> 32); } @@ -371,7 +433,7 @@ int Compare_FiTime(const CFiTime *a1, const CFiTime *a2) return 0; } -bool FILETIME_To_timespec(const FILETIME &ft, timespec &ts) +bool FILETIME_To_timespec(const FILETIME &ft, CFiTime &ts) { UInt32 quantums; const Int64 sec = NWindows::NTime::FileTime_To_UnixTime64_and_Quantums(ft, quantums); @@ -380,7 +442,7 @@ bool FILETIME_To_timespec(const FILETIME &ft, timespec &ts) if (sec2 == sec) { ts.tv_sec = sec2; - ts.tv_nsec = (long)(quantums * 100); + ts.tv_nsec = (Int32)(quantums * 100); return true; } return false; diff --git a/DOC/7zip.hhp b/DOC/7zip.hhp index 6c6bd70..58fbd1f 100644 --- a/DOC/7zip.hhp +++ b/DOC/7zip.hhp @@ -71,6 +71,7 @@ cmdline\switches\working_dir.htm cmdline\switches\exclude.htm fm\options.htm fm\benchmark.htm +fm\temp.htm fm\index.htm fm\menu.htm fm\about.htm diff --git a/DOC/7zip.wxs b/DOC/7zip.wxs index df1d320..7705d16 100644 --- a/DOC/7zip.wxs +++ b/DOC/7zip.wxs @@ -1,7 +1,7 @@ - - + + diff --git a/DOC/License.txt b/DOC/License.txt index c9e858f..8917dfc 100644 --- a/DOC/License.txt +++ b/DOC/License.txt @@ -3,14 +3,16 @@ License for use and distribution ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - 7-Zip Copyright (C) 1999-2020 Igor Pavlov. + 7-Zip Copyright (C) 1999-2024 Igor Pavlov. The licenses for files are: - 1) CPP/7zip/Compress/Rar* files: the "GNU LGPL" with "unRAR license restriction" - 2) CPP/7zip/Compress/LzfseDecoder.cpp: the "BSD 3-clause License" - 3) Some files are "public domain" files, if "public domain" status is stated in source file. - 4) the "GNU LGPL" for all other files. If there is no license information in + - CPP/7zip/Compress/Rar* files: the "GNU LGPL" with "unRAR license restriction" + - CPP/7zip/Compress/LzfseDecoder.cpp: the "BSD 3-clause License" + - C/ZstdDec.c: the "BSD 3-clause License" + - C/Xxh64.c: the "BSD 2-clause License" + - Some files are "public domain" files, if "public domain" status is stated in source file. + - the "GNU LGPL" for all other files. If there is no license information in some source file, that file is under the "GNU LGPL". The "GNU LGPL" with "unRAR license restriction" means that you must follow both @@ -19,8 +21,8 @@ - GNU LGPL information - -------------------- +GNU LGPL information +-------------------- This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public @@ -33,58 +35,117 @@ Lesser General Public License for more details. You should have received a copy of the GNU Lesser General Public - License along with this library; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + License along with this library; if not, + you can get a copy of the GNU Lesser General Public License from + http://www.gnu.org/ - BSD 3-clause License - -------------------- +BSD 3-clause License in 7-Zip code +---------------------------------- - The "BSD 3-clause License" is used for the code in LzfseDecoder.cpp that implements LZFSE data decompression. - That code was derived from the code in the "LZFSE compression library" developed by Apple Inc, - that also uses the "BSD 3-clause License": + The "BSD 3-clause License" is used for the following code in 7z.dll + 1) LZFSE data decompression. + CPP/7zip/Compress/LzfseDecoder.cpp. + That code was derived from the code in the "LZFSE compression library" developed by Apple Inc, + that also uses the "BSD 3-clause License". + 2) ZSTD data decompression. + C/ZstdDec.c + that code was developed using original zstd decoder code as reference code. + The original zstd decoder code was developed by Facebook Inc, + that also uses the "BSD 3-clause License". - ---- - Copyright (c) 2015-2016, Apple Inc. All rights reserved. + Copyright (c) 2015-2016, Apple Inc. All rights reserved. + Copyright (c) Facebook, Inc. All rights reserved. + Copyright (c) 2023-2024 Igor Pavlov. - Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: +Text of the "BSD 3-clause License" +---------------------------------- - 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: - 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the distribution. +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. - 3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived - from this software without specific prior written permission. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - ---- +3. Neither the name of the copyright holder nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +--- - unRAR license restriction - ------------------------- - The decompression engine for RAR archives was developed using source - code of unRAR program. - All copyrights to original unRAR code are owned by Alexander Roshal. - The license for original unRAR code has the following restriction: +BSD 2-clause License in 7-Zip code +---------------------------------- - The unRAR sources cannot be used to re-create the RAR compression algorithm, - which is proprietary. Distribution of modified unRAR sources in separate form - or as a part of other software is permitted, provided that it is clearly - stated in the documentation and source comments that the code may - not be used to develop a RAR (WinRAR) compatible archiver. + The "BSD 2-clause License" is used for the XXH64 code in 7-Zip. + C/Xxh64.c + XXH64 code in 7-Zip was derived from the original XXH64 code developed by Yann Collet. - -- - Igor Pavlov + Copyright (c) 2012-2021 Yann Collet. + Copyright (c) 2023-2024 Igor Pavlov. + +Text of the "BSD 2-clause License" +---------------------------------- + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +--- + + + + +unRAR license restriction +------------------------- + +The decompression engine for RAR archives was developed using source +code of unRAR program. +All copyrights to original unRAR code are owned by Alexander Roshal. + +The license for original unRAR code has the following restriction: + + The unRAR sources cannot be used to re-create the RAR compression algorithm, + which is proprietary. Distribution of modified unRAR sources in separate form + or as a part of other software is permitted, provided that it is clearly + stated in the documentation and source comments that the code may + not be used to develop a RAR (WinRAR) compatible archiver. + +-- diff --git a/DOC/Methods.txt b/DOC/Methods.txt index 541f1c1..3e5707c 100644 --- a/DOC/Methods.txt +++ b/DOC/Methods.txt @@ -1,8 +1,8 @@ 7-Zip method IDs for 7z and xz archives --------------------------------------- -Version: 23.01 -Date: 2023-06-30 +Version: 24.02 +Date: 2024-03-22 Each compression or crypto method in 7z is associated with unique binary value (ID). The length of ID in bytes is arbitrary but it can not exceed 63 bits (8 bytes). @@ -38,6 +38,7 @@ List of defined IDs 08 - ARMT (little-endian) 09 - SPARC 0A - ARM64 +0B - RISCV 21 - LZMA2 diff --git a/DOC/lzma.txt b/DOC/lzma.txt index 142feb1..0d9863c 100644 --- a/DOC/lzma.txt +++ b/DOC/lzma.txt @@ -1,6 +1,6 @@ LZMA compression ---------------- -Version: 23.01 +Version: 24.05 This file describes LZMA encoding and decoding functions written in C language. diff --git a/DOC/readme.txt b/DOC/readme.txt index 5cd90c5..a33af83 100644 --- a/DOC/readme.txt +++ b/DOC/readme.txt @@ -1,9 +1,9 @@ -7-Zip 23.01 Sources +7-Zip 24.05 Sources ------------------- 7-Zip is a file archiver for Windows. -7-Zip Copyright (C) 1999-2023 Igor Pavlov. +7-Zip Copyright (C) 1999-2024 Igor Pavlov. License Info diff --git a/DOC/src-history.txt b/DOC/src-history.txt index c1c1b71..1f29322 100644 --- a/DOC/src-history.txt +++ b/DOC/src-history.txt @@ -1,6 +1,52 @@ HISTORY of the 7-Zip source code -------------------------------- +24.05 2024-05-14 +------------------------- +- New switch -myv={MMNN} to set decoder compatibility version for 7z archive creating. + {MMNN} is 4-digit number that represents the version of 7-Zip without a dot. + If -myv={MMNN} switch is specified, 7-Zip will only use compression methods that can + be decoded by the specified version {MMNN} of 7-Zip and newer versions. + If -myv={MMNN} switch is not specified, -myv=2300 is used, and 7-Zip will only + use compression methods that can be decoded by 7-Zip 23.00 and newer versions. +- New switch -myfa={FilterID} to allow 7-Zip to use the specified filter method for 7z archive creating. +- New switch -myfd={FilterID} to disallow 7-Zip to use the specified filter method for 7z archive creating. + + +24.03 2024-03-23 +------------------------- +- 7-Zip now can use new RISCV filter for compression to 7z and xz archives. + RISCV filter can increase compression ratio for data containing executable + files compiled for RISC-V architecture. +- The speed for LZMA and LZMA2 decompression in ARM64 version for Windows + was increased by 20%-60%. + It uses arm64 assembler code, and clang-cl is required for arm64 assembler code compiling. +- Some bugs were fixed. + + +24.01 2024-01-31 +------------------------- +- 7-Zip uses file C/Precomp.h that is included to all c and c++ files. + CPP/Common/Common.h also includes C/Precomp.h. + C/Precomp.h defines the following macros (if _WIN32 is defined): + Z7_LARGE_PAGES 1 + Z7_LONG_PATH 1 + Z7_WIN32_WINNT_MIN 0x0500 (or higher) + _WIN32_WINNT 0x0500 (or higher) + WINVER _WIN32_WINNT + UNICODE 1 + _UNICODE 1 + if _WIN32_WINNT is defined already, C/Precomp.h doesn't redefine it. + +- 7-Zip now can unpack ZSTD archives (.zst extension). +- 7-Zip now can unpack ZIP and SquashFS archives that use ZSTD compression method. +- 7-Zip now supports fast hash algorithm XXH64 that is used in ZSTD. +- Speed optimizations for archive unpacking: rar, zip, gz, wim, cab. +- Speed optimizations for hash caclulation: CRC-32, CRC-64, Blake2sp. +- The bug was fixed: 7-Zip for Linux could fail for multivolume creation in some cases. +- Some bugs were fixed. + + 23.01 2023-06-20 ------------------------- - All external macros for compiling C/C++ code of 7-Zip now have Z7_ prefix.