diff --git a/3rdparty/CMakeLists.txt b/3rdparty/CMakeLists.txt index a6228b01de..705158b15a 100755 --- a/3rdparty/CMakeLists.txt +++ b/3rdparty/CMakeLists.txt @@ -149,6 +149,8 @@ if(CMAKE_TOOLCHAIN_FILE) list(APPEND NBL_JPEG_CMAKE_OPTIONS "-DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}") endif() +nbl_append_sanitize_address_cmake_options(NBL_JPEG_CMAKE_OPTIONS) + # TODO: might find an alternative library which supports add_subdirectory, untill then we need to switch to a workaround just like # we do for DXC due to: https://github.com/libjpeg-turbo/libjpeg-turbo/blob/0b742742c873025e2a127918d4969238ace7ae5b/CMakeLists.txt#L69 execute_process(COMMAND "${CMAKE_COMMAND}" -S "${CMAKE_CURRENT_SOURCE_DIR}/libjpeg-turbo" -B "${CMAKE_CURRENT_BINARY_DIR}/libjpeg-turbo" -G "${CMAKE_GENERATOR}" ${NBL_JPEG_CMAKE_OPTIONS} @@ -492,10 +494,6 @@ endif() foreach(trgt IN LISTS NBL_3RDPARTY_TARGETS) set_property(TARGET ${trgt} PROPERTY MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>$<$:DLL>") - if(MSVC AND NBL_SANITIZE_ADDRESS) - set_property(TARGET ${trgt} PROPERTY COMPILE_OPTIONS /fsanitize=address) - endif() - get_target_property(NBL_TARGET_TYPE ${trgt} TYPE) if(NOT "${NBL_TARGET_TYPE}" STREQUAL "INTERFACE_LIBRARY") # maybe explicit global mapping would be better, to discuss diff --git a/3rdparty/dxc/CMakeLists.txt b/3rdparty/dxc/CMakeLists.txt index d6ea0d0554..506dd9cc60 100644 --- a/3rdparty/dxc/CMakeLists.txt +++ b/3rdparty/dxc/CMakeLists.txt @@ -43,6 +43,8 @@ list(APPEND NBL_DXC_CMAKE_OPTIONS "-DDXC_SPIRV_TOOLS_DIR=${DXC_SPIRV_TOOLS_DIR}" list(APPEND NBL_DXC_CMAKE_OPTIONS "-DDXC_SPIRV_HEADERS_DIR=${DXC_SPIRV_HEADERS_DIR}") list(APPEND NBL_DXC_CMAKE_OPTIONS "-DDXC_ENABLE_ETW=OFF") +nbl_append_sanitize_address_cmake_options(NBL_DXC_CMAKE_OPTIONS) + if(NOT NBL_IS_MULTI_CONFIG) list(APPEND NBL_DXC_CMAKE_OPTIONS "-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}") endif() @@ -54,7 +56,9 @@ endif() #else() # NBL_EXT_P_APPEND_COMPILE_OPTIONS(NBL_DXC_CMAKE_OPTIONS Release RelWithDebInfo Debug) #endif() -#list(TRANSFORM NBL_DXC_CMAKE_OPTIONS REPLACE "/fp:fast" "/fp:precise") +if(MSVC AND NBL_SANITIZE_ADDRESS) + list(TRANSFORM NBL_DXC_CMAKE_OPTIONS REPLACE "/fp:fast" "/fp:precise") +endif() if(WIN32) if(NOT DEFINED HLSL_AUTOCRLF) diff --git a/3rdparty/openexr b/3rdparty/openexr index c8a74d9ac9..aaf5f750d7 160000 --- a/3rdparty/openexr +++ b/3rdparty/openexr @@ -1 +1 @@ -Subproject commit c8a74d9ac97dd579a47a7913f361a87349c0fffd +Subproject commit aaf5f750d7a5fd117d79932d209f0e9816cbff1f diff --git a/CMakeLists.txt b/CMakeLists.txt index 773c9c3563..c21da262c0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,6 +3,10 @@ # For conditions of distribution and use, see copyright notice in nabla.h.in or nabla.h cmake_minimum_required(VERSION 3.31..4.2.0) +if(NOT DEFINED CMAKE_POLICY_VERSION_MINIMUM) + set(CMAKE_POLICY_VERSION_MINIMUM 3.31 CACHE STRING "Minimum policy version for third-party projects") +endif() + # TODO: Yas - once we deploy 4.x we will fire `cmake_policy(VERSION [...])` instead of manually picking policies # https://cmake.org/cmake/help/latest/command/cmake_minimum_required.html#policy-version # also we should update deps which throw warnings about < 3.10 compatibility @@ -46,6 +50,7 @@ option(NBL_STATIC_BUILD "" OFF) # ON for static builds, OFF for shared option(NBL_COMPILER_DYNAMIC_RUNTIME "" ON) option(NBL_SANITIZE_ADDRESS OFF) +option(NBL_DEBUG_RTC_ENABLED "Enable Runtime Checks for Debug builds" OFF) set(CMAKE_MSVC_DEBUG_INFORMATION_FORMAT $<$:ProgramDatabase>) # ignored on non xMSVC-ABI targets diff --git a/cmake/adjust/flags.cmake b/cmake/adjust/flags.cmake index 1e67914ae0..564f5f7ff7 100644 --- a/cmake/adjust/flags.cmake +++ b/cmake/adjust/flags.cmake @@ -281,6 +281,7 @@ function(nbl_adjust_flags) # global compile options list(APPEND _D_NBL_COMPILE_OPTIONS_ ${NBL_COMPILE_OPTIONS}) + list(APPEND _D_NBL_LINK_OPTIONS_ ${NBL_LINK_OPTIONS}) foreach(CONFIG ${CMAKE_CONFIGURATION_TYPES}) string(TOUPPER "${CONFIG}" CONFIG_U) @@ -316,15 +317,21 @@ function(nbl_adjust_flags) # global compile options list(APPEND _D_NBL_COMPILE_OPTIONS_ ${NBL_COMPILE_OPTIONS}) + list(APPEND _D_NBL_LINK_OPTIONS_ ${NBL_LINK_OPTIONS}) foreach(_NBL_OPTION_IMPL_ ${_NBL_OPTIONS_IMPL_}) string(REPLACE "NBL_MAP_" "" NBL_MAP_CONFIGURATION_FROM "NBL_${_NBL_OPTION_IMPL_}") string(TOUPPER "${NBL_${_NBL_OPTION_IMPL_}}" NBL_MAP_CONFIGURATION_TO) set(NBL_TO_CONFIG_COMPILE_OPTIONS ${NBL_${NBL_MAP_CONFIGURATION_TO}_COMPILE_OPTIONS}) + set(NBL_TO_CONFIG_LINK_OPTIONS ${NBL_${NBL_MAP_CONFIGURATION_TO}_LINK_OPTIONS}) # per configuration compile options with mapping list(APPEND _D_NBL_COMPILE_OPTIONS_ $<$:${NBL_TO_CONFIG_COMPILE_OPTIONS}>) + list(APPEND _D_NBL_LINK_OPTIONS_ $<$:${NBL_TO_CONFIG_LINK_OPTIONS}>) endforeach() - set_directory_properties(PROPERTIES COMPILE_OPTIONS "${_D_NBL_COMPILE_OPTIONS_}") + set_directory_properties(PROPERTIES + COMPILE_OPTIONS "${_D_NBL_COMPILE_OPTIONS_}" + LINK_OPTIONS "${_D_NBL_LINK_OPTIONS_}" + ) endif() -endfunction() \ No newline at end of file +endfunction() diff --git a/cmake/adjust/template/vendor/impl/Clang.cmake b/cmake/adjust/template/vendor/impl/Clang.cmake index 0b00294411..984dc94681 100644 --- a/cmake/adjust/template/vendor/impl/Clang.cmake +++ b/cmake/adjust/template/vendor/impl/Clang.cmake @@ -77,11 +77,11 @@ else() endif() if(NBL_SANITIZE_ADDRESS) - NBL_REQUEST_COMPILE_OPTION_SUPPORT(LANG ${LANG} COMPILE_OPTIONS -fsanitize=address) + NBL_REQUEST_COMPILE_OPTION_SUPPORT(LANG ${LANG} COMPILE_OPTIONS -fsanitize=address LINK_OPTIONS -fsanitize=address) endif() if(NBL_SANITIZE_THREAD) - NBL_REQUEST_COMPILE_OPTION_SUPPORT(LANG ${LANG} COMPILE_OPTIONS -fsanitize=thread) + NBL_REQUEST_COMPILE_OPTION_SUPPORT(LANG ${LANG} COMPILE_OPTIONS -fsanitize=thread LINK_OPTIONS -fsanitize=thread) endif() NBL_REQUEST_COMPILE_OPTION_SUPPORT(LANG ${LANG} CONFIG DEBUG COMPILE_OPTIONS @@ -106,4 +106,4 @@ else() -mno-incremental-linker-compatible # https://clang.llvm.org/docs/ClangCommandLineReference.html#cmdoption-clang-mincremental-linker-compatible -DNDEBUG ) -endif() \ No newline at end of file +endif() diff --git a/cmake/adjust/template/vendor/impl/frontend/MSVC.cmake b/cmake/adjust/template/vendor/impl/frontend/MSVC.cmake index 5b5e8c8f50..f433b553e3 100644 --- a/cmake/adjust/template/vendor/impl/frontend/MSVC.cmake +++ b/cmake/adjust/template/vendor/impl/frontend/MSVC.cmake @@ -23,20 +23,24 @@ if(NBL_SANITIZE_ADDRESS) NBL_REQUEST_COMPILE_OPTION_SUPPORT(LANG ${LANG} COMPILE_OPTIONS /fsanitize=address # https://learn.microsoft.com/en-us/cpp/build/reference/fsanitize?view=msvc-170 ) - - NBL_REQUEST_COMPILE_OPTION_SUPPORT(LANG ${LANG} CONFIG DEBUG COMPILE_OPTIONS - /RTC1 # https://learn.microsoft.com/en-us/cpp/build/reference/rtc-run-time-error-checks?view=msvc-170 - ) endif() -NBL_REQUEST_COMPILE_OPTION_SUPPORT(LANG ${LANG} CONFIG DEBUG COMPILE_OPTIONS +set(_NBL_MSVC_DEBUG_COMPILE_OPTIONS /Ob0 # https://learn.microsoft.com/en-us/cpp/build/reference/ob-inline-function-expansion?view=msvc-170 /Od # https://learn.microsoft.com/en-us/cpp/build/reference/od-disable-debug?view=msvc-170 /Oy- # https://learn.microsoft.com/en-us/cpp/build/reference/oy-frame-pointer-omission?view=msvc-170 +) +if(NBL_DEBUG_RTC_ENABLED AND NOT NBL_SANITIZE_ADDRESS) + list(APPEND _NBL_MSVC_DEBUG_COMPILE_OPTIONS /RTC1) +endif() + +NBL_REQUEST_COMPILE_OPTION_SUPPORT(LANG ${LANG} CONFIG DEBUG COMPILE_OPTIONS + ${_NBL_MSVC_DEBUG_COMPILE_OPTIONS} LINK_OPTIONS /INCREMENTAL # https://learn.microsoft.com/en-us/cpp/build/reference/incremental-link-incrementally?view=msvc-170 ) +unset(_NBL_MSVC_DEBUG_COMPILE_OPTIONS) NBL_REQUEST_COMPILE_OPTION_SUPPORT(LANG ${LANG} CONFIG RELEASE COMPILE_OPTIONS /O2 # https://learn.microsoft.com/en-us/cpp/build/reference/o1-o2-minimize-size-maximize-speed?view=msvc-170 @@ -63,4 +67,4 @@ NBL_REQUEST_COMPILE_OPTION_SUPPORT(LANG ${LANG} CONFIG RELWITHDEBINFO COMPILE_OP LINK_OPTIONS /INCREMENTAL # https://learn.microsoft.com/en-us/cpp/build/reference/incremental-link-incrementally?view=msvc-170 -) \ No newline at end of file +) diff --git a/cmake/common.cmake b/cmake/common.cmake index 2de6dc758f..19bad18182 100755 --- a/cmake/common.cmake +++ b/cmake/common.cmake @@ -16,6 +16,34 @@ include_guard(GLOBAL) include(ProcessorCount) +# tmp for external projects, to be removed later when I get rid of them (dxc + jpeg currently) +function(nbl_append_sanitize_address_cmake_options out_list) + if(NOT NBL_SANITIZE_ADDRESS) + return() + endif() + + if(MSVC) + set(_NBL_ASAN_FLAG "/fsanitize=address") + else() + set(_NBL_ASAN_FLAG "-fsanitize=address") + set(_NBL_ASAN_LINK_SUFFIX " ${_NBL_ASAN_FLAG}") + endif() + + list(APPEND ${out_list} + "-DCMAKE_C_FLAGS:STRING=${CMAKE_C_FLAGS} ${_NBL_ASAN_FLAG}" + "-DCMAKE_CXX_FLAGS:STRING=${CMAKE_CXX_FLAGS} ${_NBL_ASAN_FLAG}" + ) + if(DEFINED _NBL_ASAN_LINK_SUFFIX) + list(APPEND ${out_list} + "-DCMAKE_EXE_LINKER_FLAGS:STRING=${CMAKE_EXE_LINKER_FLAGS}${_NBL_ASAN_LINK_SUFFIX}" + "-DCMAKE_SHARED_LINKER_FLAGS:STRING=${CMAKE_SHARED_LINKER_FLAGS}${_NBL_ASAN_LINK_SUFFIX}" + ) + endif() + unset(_NBL_ASAN_FLAG) + unset(_NBL_ASAN_LINK_SUFFIX) + set(${out_list} "${${out_list}}" PARENT_SCOPE) +endfunction() + # Macro creating project for an executable # Project and target get its name from directory when this macro gets executed (truncating number in the beginning of the name and making all lower case) # Created because of common cmake code for examples and tools diff --git a/examples_tests b/examples_tests index 671d1f16b0..55afb97d73 160000 --- a/examples_tests +++ b/examples_tests @@ -1 +1 @@ -Subproject commit 671d1f16b0837a70c3016c2472864528f35db0bc +Subproject commit 55afb97d734ac99a95c92da526244cd9990cd905 diff --git a/src/nbl/CMakeLists.txt b/src/nbl/CMakeLists.txt index 512633536f..abadd07912 100644 --- a/src/nbl/CMakeLists.txt +++ b/src/nbl/CMakeLists.txt @@ -394,6 +394,9 @@ nbl_adjust_flags(TARGET Nabla MAP_RELEASE Release MAP_RELWITHDEBINFO RelWithDebI nbl_adjust_definitions() option(NBL_WAVE_STRING_RESOLVER_TU_DEBUG_OPTIMISATION "Enable to optimise CWaveStringResolver.cpp in Debug configuration, uses RWDI compile options for the TU" ON) +if(NBL_SANITIZE_ADDRESS OR NBL_DEBUG_RTC_ENABLED) + set(NBL_WAVE_STRING_RESOLVER_TU_DEBUG_OPTIMISATION OFF CACHE BOOL "" FORCE) +endif() if(NBL_WAVE_STRING_RESOLVER_TU_DEBUG_OPTIMISATION) set_source_files_properties(asset/utils/CWaveStringResolver.cpp PROPERTIES # just enabling inlining and optimisations will help a lot @@ -868,4 +871,4 @@ source_group(TREE "${NBL_ROOT_PATH}" source_group(TREE "${NBL_ROOT_PATH}" PREFIX "Source Files" FILES ${NABLA_SOURCE_FILES} -) \ No newline at end of file +) diff --git a/src/nbl/builtin/utils.cmake b/src/nbl/builtin/utils.cmake index 6465c2ac6d..f8cdafab77 100644 --- a/src/nbl/builtin/utils.cmake +++ b/src/nbl/builtin/utils.cmake @@ -253,7 +253,4 @@ function(ADD_CUSTOM_BUILTIN_RESOURCES _TARGET_NAME_ _BUNDLE_NAME_ _BUNDLE_SEARCH _ADD_PROPERTY_(BUILTIN_RESOURCES_HEADERS NBL_BUILTIN_RESOURCES_HEADERS) _ADD_PROPERTY_(BUILTIN_RESOURCES_INCLUDE_SEARCH_DIRECTORY _OUTPUT_INCLUDE_SEARCH_DIRECTORY_CONFIG) - if(MSVC AND NBL_SANITIZE_ADDRESS) - set_property(TARGET ${_TARGET_NAME_} PROPERTY COMPILE_OPTIONS /fsanitize=address) - endif() endfunction() diff --git a/src/nbl/system/CArchiveLoaderZip.cpp b/src/nbl/system/CArchiveLoaderZip.cpp index 9f22e60790..c820f12cb6 100644 --- a/src/nbl/system/CArchiveLoaderZip.cpp +++ b/src/nbl/system/CArchiveLoaderZip.cpp @@ -20,9 +20,7 @@ struct SZIPFileCentralDirFileHeader uint16_t CompressionMethod; uint16_t LastModFileTime; uint16_t LastModFileDate; - uint32_t CRC32; - uint32_t CompressedSize; - uint32_t UncompressedSize; + nbl::system::CArchiveLoaderZip::SZIPFileDataDescriptor DataDescriptor; uint16_t FilenameLength; uint16_t ExtraFieldLength; uint16_t FileCommentLength; @@ -35,7 +33,16 @@ struct SZIPFileCentralDirFileHeader // extra field (variable size) // file comment (variable size) + static constexpr uint32_t ExpectedSignature = 0x02014b50u; + + size_t calcSize() const + { + return sizeof(SZIPFileCentralDirFileHeader) + FilenameLength + ExtraFieldLength + FileCommentLength; + } } PACK_STRUCT; + +static_assert(sizeof(SZIPFileCentralDirFileHeader) == 46); + struct SZIPFileCentralDirEnd { static inline constexpr uint32_t ExpectedSig = 0x06054b50u; @@ -65,9 +72,13 @@ struct SGZIPMemberHeader uint32_t time; uint8_t extraFlags; // slow compress = 2, fast compress = 4 uint8_t operatingSystem; + + static constexpr uint16_t ExpectedSignature = 0x8b1fu; } PACK_STRUCT; #include "nbl/nblunpack.h" +static_assert(sizeof(SGZIPMemberHeader) == 10); + enum E_GZIP_FLAGS { EGZF_TEXT_DAT = 1, @@ -92,280 +103,333 @@ constexpr int16_t ZIP_INFO_IN_DATA_DESCRIPTOR = 0x0008; using namespace nbl; using namespace nbl::system; - -core::smart_refctd_ptr CArchiveLoaderZip::createArchive_impl(core::smart_refctd_ptr&& file, const std::string_view& password) const +core::smart_refctd_ptr CArchiveLoaderZip::createArchiveFromGZIP(core::smart_refctd_ptr&& file, const std::string_view& password) const { - if (!file) - return nullptr; - - uint16_t sig; - { - IFile::success_t success; - file->read(success,&sig,0,sizeof(sig)); - if (!success) - return nullptr; - } - std::shared_ptr> items = std::make_shared>(); core::vector itemsMetadata; - // load file entries - { - const bool isGZip = sig==0x8b1fu; - // - auto addItem = [&items,&itemsMetadata](const std::string& _path, const size_t offset, const SZIPFileHeader& meta) -> void - { - // we need to have a filename or we skip - if (_path.empty()) - return; - - auto& item = items->emplace_back(); - item.pathRelativeToArchive = _path; - item.size = meta.DataDescriptor.UncompressedSize; - item.offset = offset; - item.ID = itemsMetadata.size(); - item.allocatorType = meta.CompressionMethod ? IFileArchive::EAT_VIRTUAL_ALLOC:IFileArchive::EAT_NULL; - itemsMetadata.push_back(meta); - }; - - // - size_t offset = 0ull; - auto readStringFromFile = [&file,&offset](auto charCallback) -> bool + items->reserve(1u); + itemsMetadata.reserve(1u); + auto addItem = [&items, &itemsMetadata](const std::string& _path, const size_t offset, const SZIPFileHeader& meta) -> void + { + // we need to have a filename or we skip + if (_path.empty()) + return; + + auto& item = items->emplace_back(); + item.pathRelativeToArchive = _path; + item.size = meta.DataDescriptor.UncompressedSize; + item.offset = offset; + item.ID = itemsMetadata.size(); + item.allocatorType = meta.CompressionMethod ? IFileArchive::EAT_VIRTUAL_ALLOC : IFileArchive::EAT_NULL; + itemsMetadata.push_back(meta); + }; + + std::string filename; + size_t gzipFileOffset = 0ull; + auto readStringFromFile = [&file, &gzipFileOffset](auto charCallback) -> bool { char c = 0x45; // make sure we start with non-zero char while (c) { IFile::success_t success; - file->read(success,&c,offset,sizeof(c)); + file->read(success, &c, gzipFileOffset, sizeof(c)); if (!success) return false; - offset += success.getBytesToProcess(); + gzipFileOffset += success.getBytesToProcess(); charCallback(c); } // if string is not null terminated, something went wrong reading the file return !c; }; - - // - std::string filename; - filename.reserve(ISystem::MAX_FILENAME_LENGTH); - if (isGZip) + + SGZIPMemberHeader gzipHeader; + { + IFile::success_t success; + file->read(success, &gzipHeader, gzipFileOffset, sizeof(gzipHeader)); + if (!success) + return nullptr; + gzipFileOffset += sizeof(gzipHeader); + } + + //! The gzip file format seems to think that there can be multiple files in a gzip file + //! TODO: But OLD Irrlicht Impl doesn't honor it!? + if (gzipHeader.sig != SGZIPMemberHeader::ExpectedSignature) + return nullptr; + + // now get the file info + if (gzipHeader.flags & EGZF_EXTRA_FIELDS) + { + // read lenth of extra data + uint16_t dataLen; + IFile::success_t success; + file->read(success, &dataLen, gzipFileOffset, sizeof(dataLen)); + if (!success) + return nullptr; + gzipFileOffset += success.getBytesToProcess(); + // skip the extra data + gzipFileOffset += dataLen; + } + // + if (gzipHeader.flags & EGZF_FILE_NAME) + { + filename.clear(); + if (!readStringFromFile([&](const char c) {filename.push_back(c); })) + return nullptr; + } + // + if (gzipHeader.flags & EGZF_COMMENT) + { + if (!readStringFromFile([](const char c) {})) + return nullptr; + } + // skip crc16 + if (gzipHeader.flags & EGZF_CRC16) + gzipFileOffset += 2; + + + SZIPFileHeader header{}; + header.FilenameLength = filename.length(); + header.CompressionMethod = gzipHeader.compressionMethod; + header.DataDescriptor.CompressedSize = file->getSize() - (gzipFileOffset + sizeof(uint64_t)); + + const size_t itemOffset = gzipFileOffset; + + gzipFileOffset += header.DataDescriptor.CompressedSize; + // read CRC + { + IFile::success_t success; + file->read(success, &header.DataDescriptor.CRC32, gzipFileOffset, sizeof(header.DataDescriptor.CRC32)); + if (!success) + return nullptr; + gzipFileOffset += success.getBytesToProcess(); + } + // read uncompressed size + { + IFile::success_t success; + file->read(success, &header.DataDescriptor.UncompressedSize, gzipFileOffset, sizeof(header.DataDescriptor.UncompressedSize)); + if (!success) + return nullptr; + gzipFileOffset += success.getBytesToProcess(); + } + + // + addItem(filename, itemOffset, header); + + assert(items->size() == itemsMetadata.size()); + if (items->empty()) + return nullptr; + + return core::make_smart_refctd_ptr(std::move(file), core::smart_refctd_ptr(m_logger.get()), items, std::move(itemsMetadata)); +} +core::smart_refctd_ptr CArchiveLoaderZip::createArchiveFromZIP(core::smart_refctd_ptr&& file, const std::string_view& password) const +{ + const size_t fileSize = file->getSize(); + if (fileSize < sizeof(SZIPFileCentralDirEnd)) + return nullptr; + + SZIPFileCentralDirEnd dirEnd; + { + dirEnd.Sig = 0u; + constexpr size_t kMaxZipCommentSize = 0xffffu; + size_t endOfCentralDirectoryOffset = fileSize - sizeof(SZIPFileCentralDirEnd); + const size_t minEndOffset = (fileSize > sizeof(SZIPFileCentralDirEnd) + kMaxZipCommentSize) ? (fileSize - sizeof(SZIPFileCentralDirEnd) - kMaxZipCommentSize) : 0u; + bool found = false; + while (true) { - SGZIPMemberHeader gzipHeader; + IFile::success_t success; + file->read(success, &dirEnd, endOfCentralDirectoryOffset, sizeof(dirEnd)); + if (success && dirEnd.Sig == SZIPFileCentralDirEnd::ExpectedSig) { - IFile::success_t success; - file->read(success,&gzipHeader,0ull,sizeof(gzipHeader)); - if (!success) - return nullptr; - offset += success.getBytesToProcess(); + found = true; + break; } + if (endOfCentralDirectoryOffset == minEndOffset) + break; + --endOfCentralDirectoryOffset; + } + if (!found) + return nullptr; + } + + // multiple disks are not supported + if (dirEnd.NumberDisk != 0 || dirEnd.NumberStart != 0) + { + assert(false); + return nullptr; + } + if (dirEnd.Offset > fileSize || dirEnd.Size > fileSize - dirEnd.Offset) + return nullptr; - //! The gzip file format seems to think that there can be multiple files in a gzip file - //! TODO: But OLD Irrlicht Impl doesn't honor it!? - if (gzipHeader.sig!=0x8b1fu) + std::shared_ptr> items = std::make_shared>(); + core::vector itemsMetadata; + + items->reserve(dirEnd.TotalEntries); + itemsMetadata.reserve(dirEnd.TotalEntries); + auto addItem = [&items, &itemsMetadata](const std::string& _path, const size_t offset, const SZIPFileHeader& meta) -> void + { + // we need to have a filename or we skip + if (_path.empty()) + return; + + auto& item = items->emplace_back(); + item.pathRelativeToArchive = _path; + item.size = meta.DataDescriptor.UncompressedSize; + item.offset = offset; + item.ID = itemsMetadata.size(); + item.allocatorType = meta.CompressionMethod ? IFileArchive::EAT_VIRTUAL_ALLOC : IFileArchive::EAT_NULL; + itemsMetadata.push_back(meta); + }; + + size_t centralDirectoryOffset = dirEnd.Offset; + for (int i = 0; i < dirEnd.TotalEntries; ++i) + { + SZIPFileCentralDirFileHeader centralDirectoryHeader; + { + if (centralDirectoryOffset > fileSize || fileSize - centralDirectoryOffset < sizeof(SZIPFileCentralDirFileHeader)) return nullptr; - - // now get the file info - if (gzipHeader.flags&EGZF_EXTRA_FIELDS) - { - // read lenth of extra data - uint16_t dataLen; - IFile::success_t success; - file->read(success,&dataLen,offset,sizeof(dataLen)); - if (!success) - return nullptr; - offset += success.getBytesToProcess(); - // skip the extra data - offset += dataLen; - } - // - if (gzipHeader.flags&EGZF_FILE_NAME) - { - filename.clear(); - if (!readStringFromFile([&](const char c){filename.push_back(c);})) - return nullptr; - } - // - if (gzipHeader.flags&EGZF_COMMENT) - { - if (!readStringFromFile([](const char c){})) - return nullptr; - } - // skip crc16 - if (gzipHeader.flags&EGZF_CRC16) - offset += 2; + IFile::success_t success; + file->read(success, ¢ralDirectoryHeader, centralDirectoryOffset, sizeof(SZIPFileCentralDirFileHeader)); + if (!success) + return nullptr; + } + if (centralDirectoryHeader.Sig != SZIPFileCentralDirFileHeader::ExpectedSignature) + { + // .zip file is corrupted + assert(false); + return nullptr; + } - SZIPFileHeader header; - memset(&header,0,sizeof(SZIPFileHeader)); - header.FilenameLength = filename.length(); - header.CompressionMethod = gzipHeader.compressionMethod; - header.DataDescriptor.CompressedSize = file->getSize()-(offset+sizeof(uint64_t)); + const size_t centralHeaderSize = centralDirectoryHeader.calcSize(); + if (centralHeaderSize < sizeof(SZIPFileCentralDirFileHeader)) + return nullptr; + if (centralDirectoryOffset + centralHeaderSize > fileSize) + return nullptr; + centralDirectoryOffset += centralHeaderSize; - const size_t itemOffset = offset; - - offset += header.DataDescriptor.CompressedSize; - // read CRC - { - IFile::success_t success; - file->read(success,&header.DataDescriptor.CRC32,offset,sizeof(header.DataDescriptor.CRC32)); - if (!success) - return nullptr; - offset += success.getBytesToProcess(); - } - // read uncompressed size - { - IFile::success_t success; - file->read(success,&header.DataDescriptor.UncompressedSize,offset,sizeof(header.DataDescriptor.UncompressedSize)); - if (!success) - return nullptr; - offset += success.getBytesToProcess(); - } + SZIPFileHeader localFileHeader; + { + const size_t localHeaderOffset = centralDirectoryHeader.RelativeOffsetOfLocalHeader; + if (localHeaderOffset > fileSize || fileSize - localHeaderOffset < sizeof(SZIPFileHeader)) + return nullptr; + IFile::success_t success; + file->read(success, &localFileHeader, localHeaderOffset, sizeof(SZIPFileHeader)); + if (!success) + return nullptr; + } + if (localFileHeader.Sig != SZIPFileHeader::ExpectedSignature) + return nullptr; + const size_t localHeaderSize = localFileHeader.calcSize(); + if (localHeaderSize < sizeof(SZIPFileHeader)) + return nullptr; + if (centralDirectoryHeader.RelativeOffsetOfLocalHeader + localHeaderSize > fileSize) + return nullptr; - // - addItem(filename,itemOffset,header); + std::string filename; + filename.resize(localFileHeader.FilenameLength); + { + IFile::success_t success; + const size_t filenameOffset = centralDirectoryHeader.RelativeOffsetOfLocalHeader + sizeof(SZIPFileHeader); + file->read(success, filename.data(), filenameOffset, localFileHeader.FilenameLength); + // TODO: assertion + if (!success) + return nullptr; } - else + + // AES encryption + if ((localFileHeader.GeneralBitFlag & ZIP_FILE_ENCRYPTED) && (localFileHeader.CompressionMethod == 99)) { - while (true) + SZipFileExtraHeader extraHeader; + SZipFileAESExtraData data; + + size_t localOffset = centralDirectoryHeader.RelativeOffsetOfLocalHeader + sizeof(SZIPFileHeader) + localFileHeader.FilenameLength; + size_t offset = localOffset + localFileHeader.ExtraFieldLength; + while (localOffset + sizeof(extraHeader) <= offset) { - SZIPFileHeader zipHeader; { IFile::success_t success; - file->read(success,&zipHeader,offset,sizeof(zipHeader)); + file->read(success, &extraHeader, localOffset, sizeof(extraHeader)); if (!success) break; - offset += success.getBytesToProcess(); + localOffset += sizeof(extraHeader); } - if (zipHeader.Sig!=0x04034b50u) + if (extraHeader.Size < 0) + break; + const size_t extraSize = static_cast(extraHeader.Size); + if (extraSize == 0) + break; + if (localOffset + extraSize > offset) break; - filename.resize(zipHeader.FilenameLength); + if (extraHeader.ID != 0x9901u) { - IFile::success_t success; - file->read(success,filename.data(),offset,zipHeader.FilenameLength); - if (!success) - break; - offset += success.getBytesToProcess(); + localOffset += extraSize; + continue; } - // AES encryption - if ((zipHeader.GeneralBitFlag&ZIP_FILE_ENCRYPTED) && (zipHeader.CompressionMethod==99)) + if (extraSize < sizeof(SZipFileAESExtraData)) + break; { - SZipFileExtraHeader extraHeader; - SZipFileAESExtraData data; - - size_t localOffset = offset; - offset += zipHeader.ExtraFieldLength; - while (true) - { - { - IFile::success_t success; - file->read(success,&extraHeader,localOffset,sizeof(extraHeader)); - if (!success) - break; - localOffset += success.getBytesToProcess(); - if (localOffset>offset) - break; - } - - if (extraHeader.ID!=0x9901u) - continue; - - { - IFile::success_t success; - file->read(success,&data,localOffset,sizeof(data)); - if (!success) - break; - localOffset += success.getBytesToProcess(); - if (localOffset>offset) - break; - } - if (data.Vendor[0]=='A' && data.Vendor[1]=='E') - { - #ifdef _NBL_COMPILE_WITH_ZIP_ENCRYPTION_ - // encode values into Sig - // AE-Version | Strength | ActualMode - zipHeader.Sig = - ((data.Version & 0xff) << 24) | - (data.EncryptionStrength << 16) | - (data.CompressionMode); - break; - #else - filename.clear(); // no support, can't decrypt - #endif - } - } + IFile::success_t success; + file->read(success, &data, localOffset, sizeof(data)); + if (!success) + break; } - else - offset += zipHeader.ExtraFieldLength; - - // if bit 3 was set, use CentralDirectory for setup - if (zipHeader.GeneralBitFlag&ZIP_INFO_IN_DATA_DESCRIPTOR) + if (data.Vendor[0] == 'A' && data.Vendor[1] == 'E') { - SZIPFileCentralDirEnd dirEnd; - dirEnd.Sig = 0u; - - // First place where the end record could be stored - offset = file->getSize()-sizeof(SZIPFileCentralDirEnd)+1ull; - while (dirEnd.Sig!=SZIPFileCentralDirEnd::ExpectedSig) - { - IFile::success_t success; - file->read(success,&dirEnd,--offset,sizeof(dirEnd)); - if (!success) - return nullptr; - } - items->reserve(dirEnd.TotalEntries); - itemsMetadata.reserve(dirEnd.TotalEntries); - offset = dirEnd.Offset; - #if 0 - while (scanCentralDirectoryHeader(offset)) {} - #endif - assert(false); // if you ever hit this, msg @devsh +#ifdef _NBL_COMPILE_WITH_ZIP_ENCRYPTION_ + // encode values into Sig + // AE-Version | Strength | ActualMode + localFileHeader.Sig = + ((data.Version & 0xff) << 24) | + (data.EncryptionStrength << 16) | + (data.CompressionMode); break; +#else + filename.clear(); // no support, can't decrypt +#endif } - - addItem(filename,offset,zipHeader); - // move forward length of data - offset += zipHeader.DataDescriptor.CompressedSize; + localOffset += extraSize; } } + + // copying the data descriptor from the central directory header because it is always valid (data descriptor from local file header may be invalid when bit 3 in general purpose bit flag is set) + localFileHeader.DataDescriptor = centralDirectoryHeader.DataDescriptor; + + const size_t fileDataOffset = centralDirectoryHeader.RelativeOffsetOfLocalHeader + localFileHeader.calcSize(); + addItem(filename, fileDataOffset, localFileHeader); } - assert(items->size()==itemsMetadata.size()); + assert(items->size() == itemsMetadata.size()); if (items->empty()) return nullptr; - return core::make_smart_refctd_ptr(std::move(file),core::smart_refctd_ptr(m_logger.get()), items, std::move(itemsMetadata)); + return core::make_smart_refctd_ptr(std::move(file), core::smart_refctd_ptr(m_logger.get()), items, std::move(itemsMetadata)); } -#if 0 -bool CFileArchiveZip::scanCentralDirectoryHeader(size_t& offset) +core::smart_refctd_ptr CArchiveLoaderZip::createArchive_impl(core::smart_refctd_ptr&& file, const std::string_view& password) const { - std::filesystem::path ZipFileName = ""; - SZIPFileCentralDirFileHeader entry; + if (!file) + return nullptr; + + uint16_t sig; + IFile::success_t success; + file->read(success, &sig, 0, sizeof(sig)); + if (!success) + return nullptr; + + const bool isGZIP = sig == SGZIPMemberHeader::ExpectedSignature; + if (isGZIP) { - system::future fut; - m_file->read(fut, &entry, offset, sizeof(SZIPFileCentralDirFileHeader)); - fut.get(); - offset += sizeof(SZIPFileCentralDirFileHeader); + return createArchiveFromGZIP(std::move(file), password); + } + else + { + return createArchiveFromZIP(std::move(file), password); } - - if (entry.Sig != 0x02014b50) - return false; // central dir headers end here. - - const long pos = offset; - offset = entry.RelativeOffsetOfLocalHeader; - scanZipHeader(offset, true); - offset = pos + entry.FilenameLength + entry.ExtraFieldLength + entry.FileCommentLength; - m_fileInfo.back().header.DataDescriptor.CompressedSize = entry.CompressedSize; - m_fileInfo.back().header.DataDescriptor.UncompressedSize = entry.UncompressedSize; - m_fileInfo.back().header.DataDescriptor.CRC32 = entry.CRC32; - m_files.back().size = entry.UncompressedSize; - return true; } -#endif CFileArchive::file_buffer_t CArchiveLoaderZip::CArchive::getFileBuffer(const IFileArchive::SFileList::found_t& item) { @@ -620,4 +684,4 @@ namespace void SzFree(void *p, void *address) { p = p; _NBL_ALIGNED_FREE(address); } ISzAlloc lzmaAlloc = { SzAlloc, SzFree }; } -#endif \ No newline at end of file +#endif diff --git a/src/nbl/system/CArchiveLoaderZip.h b/src/nbl/system/CArchiveLoaderZip.h index 0a57b62ec9..1524bbe22d 100644 --- a/src/nbl/system/CArchiveLoaderZip.h +++ b/src/nbl/system/CArchiveLoaderZip.h @@ -31,8 +31,18 @@ class CArchiveLoaderZip final : public IArchiveLoader int16_t ExtraFieldLength; // filename (variable size) // extra field (variable size ) + + static constexpr uint32_t ExpectedSignature = 0x04034b50u; + + size_t calcSize() const + { + return sizeof(SZIPFileHeader) + FilenameLength + ExtraFieldLength; + } } PACK_STRUCT; #include "nbl/nblunpack.h" + + static_assert(sizeof(SZIPFileHeader) == 30); + class CArchive final : public CFileArchive { public: @@ -75,6 +85,8 @@ class CArchiveLoaderZip final : public IArchiveLoader private: core::smart_refctd_ptr createArchive_impl(core::smart_refctd_ptr&& file, const std::string_view& password) const override; + core::smart_refctd_ptr createArchiveFromGZIP(core::smart_refctd_ptr&& file, const std::string_view& password) const; + core::smart_refctd_ptr createArchiveFromZIP(core::smart_refctd_ptr&& file, const std::string_view& password) const; }; }