From 41e166a89f65222e77d57ebdd267e062402d3566 Mon Sep 17 00:00:00 2001 From: qaate47 Date: Wed, 8 Jan 2025 19:31:16 +0100 Subject: [PATCH] bo4 .ff gscc dumper --- src/acts/tools/fastfiles.cpp | 315 ++++++++++++++++++++++++++++++----- src/acts/tools/test.cpp | 17 ++ src/shared/deps/oodle.hpp | 99 ++++++++++- 3 files changed, 387 insertions(+), 44 deletions(-) diff --git a/src/acts/tools/fastfiles.cpp b/src/acts/tools/fastfiles.cpp index a8d67a5..3b51a72 100644 --- a/src/acts/tools/fastfiles.cpp +++ b/src/acts/tools/fastfiles.cpp @@ -1,4 +1,6 @@ #include +#include +#include namespace fastfiles { int ComputeChecksum32(const char* buffer, unsigned int len, int start) { @@ -114,11 +116,40 @@ namespace { } out << "\n"; } + enum XFileCompression : byte { + XFILE_UNCOMPRESSED = 0x0, + XFILE_ZLIB = 0x1, + XFILE_ZLIB_HC = 0x2, + XFILE_LZ4 = 0x3, + XFILE_LZ4_HC = 0x4, + XFILE_BDELTA_UNCOMP = 0x5, + XFILE_BDELTA_ZLIB = 0x6, + XFILE_BDELTA_LZMA = 0x7, + XFILE_OODLE_KRAKEN = 0x8, + XFILE_OODLE_MERMAID = 0x9, + XFILE_OODLE_SELKIE = 0xA, + XFILE_OODLE_LZNA = 0xB, + XFILE_COMPRESSION_COUNT = 0xC, + }; + const char* CompressionNames[]{ + "none", + "zlib", + "zlib_hc", + "lz4", + "lz4_hc", + "bdelta_uncomp", + "bdelta_zlib", + "bdelta_lzma", + "oodle_kraken", + "oodle_mermaid", + "oodle_selkie", + "oodle_lzna", + }; struct XFile { uint8_t magic[8]; uint32_t version; uint8_t server; - uint8_t compression; + XFileCompression compression; uint8_t platform; uint8_t encrypted; uint64_t timestamp; @@ -138,7 +169,22 @@ namespace { byte pad0[184]; char key[8]; byte pad1[328]; + }; static_assert(sizeof(XFile) == 0x840); + struct XFileFD { + XFile header; + XFile newHeader; + uint64_t headerSize; + uint64_t unk1088; + uint64_t unk1090; + uint64_t unk1098; + uint64_t unk10a0; }; + struct DBStreamHeader { + uint32_t compressedSize; + uint32_t uncompressedSize; + uint32_t alignedSize; + uint32_t offset; + }; static_assert(sizeof(DBStreamHeader) == 16); struct aesKey_t { uint32_t fileNameHash; uint32_t version; @@ -169,80 +215,267 @@ namespace { { "1080_wz_common", 0x1, { 0x88, 0x85, 0x7a, 0xe5, 0x22, 0x0a, 0x36, 0xf9, 0x76, 0xe2, 0x18, 0xe8, 0x7b, 0x7b, 0xbc, 0x81, 0xeb, 0x8f, 0x5d, 0xa2, 0x7c, 0xad, 0x1d, 0x7c, 0x77, 0x2e, 0xc6, 0xbb, 0x12, 0xbd, 0xd7, 0xa0 } }, }; - int fftest(Process& proc, int argc, const char* argv[]) { + int fftest(int argc, const char* argv[]) { if (tool::NotEnoughParam(argc, 1)) { return tool::BAD_USAGE; } - std::string fileBuff{}; + std::vector fileBuff{}; if (!utils::ReadFile(argv[2], fileBuff)) { LOG_ERROR("Can't read file {}", argv[2]); return tool::BASIC_ERROR; } + std::filesystem::path deco{ argv[2] }; + + std::string nnext{ deco.extension().string() + ".dec"}; + bool isfd = deco.extension() == ".fd"; + deco.replace_extension(nnext); + + core::bytebuffer::ByteBuffer reader{ fileBuff }; + + int ret{}; + auto PrintXFile = [&ret, &reader](XFile* buffer) { + if (*reinterpret_cast(buffer->magic) != 0x3030303066664154) { + LOG_ERROR("Invalid FF magic: {:x}", *reinterpret_cast(buffer)); + ret = tool::BASIC_ERROR; + return; + } - XFile* buffer = reinterpret_cast(fileBuff.data()); + if (buffer->version != 0x27F) { + LOG_ERROR("Not a T8 FF"); + ret = tool::BASIC_ERROR; + return; + } - if (fileBuff.size() < sizeof(*buffer)) { - LOG_ERROR("FF too small for header"); - return tool::BASIC_ERROR; + + if (buffer->encrypted) { + auto checksum32 = fastfiles::ComputeChecksum32(buffer->key, (unsigned int)strlen(buffer->key), 0); + LOG_INFO("checksum32 . {}", checksum32); + } + + if (buffer->compression >= XFILE_COMPRESSION_COUNT) { + LOG_ERROR("Not implemented for this compression type: {}", (int)buffer->compression); + ret = tool::BASIC_ERROR; + return; + } + + LOG_INFO("server ............. {}", (buffer->server ? "true" : "false")); + LOG_INFO("encrypted .......... {}", (buffer->encrypted ? "true" : "false")); + LOG_INFO("compress ........... {} (0x{:x})", CompressionNames[buffer->compression], (int)buffer->compression); + LOG_INFO("platform ........... {}", (int)buffer->platform); + LOG_INFO("builder ............ {}", buffer->builder); + LOG_INFO("timestamp .......... {}", buffer->timestamp); + + for (size_t i = 0; i < 4; i++) { + LOG_INFO("archiveChecksum[{}] . {:x}", i, buffer->archiveChecksum[i]); + } + + LOG_INFO("key ................ {}", buffer->key); + }; + + LOG_INFO("size ............... {}B (0x{:x})", utils::FancyNumber(fileBuff.size()), fileBuff.size()); + XFile* buffer; + if (isfd) { + XFileFD* fd{ reader.ReadPtr() }; + buffer = &fd->header; + PrintXFile(&fd->header); + LOG_INFO("--------------"); + PrintXFile(&fd->newHeader); + + if (fd->headerSize != sizeof(XFileFD)) { + LOG_ERROR("Invalid fd header size: 0x{:x} != 0x{:x}", fd->headerSize, sizeof(XFileFD)); + return tool::BASIC_ERROR; + } } - if (*reinterpret_cast(buffer->magic) != 0x3030303066664154) { - LOG_ERROR("Invalid FF magic: {:x}", *reinterpret_cast(buffer)); - return tool::BASIC_ERROR; + else { + buffer = reader.ReadPtr(); + PrintXFile(buffer); } + if (ret) return ret; - if (buffer->version != 0x27F) { - LOG_ERROR("Not a T8 FF"); - return tool::BASIC_ERROR; + LOG_INFO("Reading into {}", deco.string()); + + deps::oodle::Oodle oodle{ deps::oodle::OO2CORE_6 }; + + if (!oodle) return tool::BASIC_ERROR; + + size_t idx{}; + + std::vector ffdata{}; + while (true) { + size_t loc{ reader.Loc() }; + + DBStreamHeader* block{ reader.ReadPtr() }; + if (!block->alignedSize) break; // end + + if (block->offset != loc) { + LOG_ERROR("bad block position: 0x{:x} != 0x{:x}", loc, block->offset); + break; + } + + + if (!block->uncompressedSize) { + reader.Align(0x800000); + continue; + } + + byte* blockBuff{ reader.ReadPtr(block->alignedSize) }; + + + LOG_TRACE("Decompressing block 0x{:x} (0x{:x}/0x{:x} -> 0x{:x})", loc, block->compressedSize, block->alignedSize, block->uncompressedSize); + + auto decompressed{ std::make_unique(block->uncompressedSize)}; + + switch (buffer->compression) { + case XFILE_UNCOMPRESSED: + if (block->uncompressedSize > block->compressedSize) { + throw std::runtime_error(std::format("Can't decompress block, decompressed size isn't big enough: 0x{:x} != 0x{:x}", block->compressedSize, block->uncompressedSize)); + } + memcpy(decompressed.get(), blockBuff, block->uncompressedSize); + break; + case XFILE_ZLIB: + case XFILE_ZLIB_HC: { + + uLongf sizef = (uLongf)block->uncompressedSize; + uLongf sizef2{ (uLongf)block->compressedSize }; + int ret; + if (ret = uncompress2(decompressed.get(), &sizef, reader.Ptr(), &sizef2) < 0) { + throw std::runtime_error(std::format("error when decompressing {}", zError(ret))); + } + break; + } + case XFILE_OODLE_KRAKEN: + case XFILE_OODLE_MERMAID: + case XFILE_OODLE_SELKIE: + case XFILE_OODLE_LZNA: { + int ret{ oodle.Decompress(blockBuff, block->compressedSize, decompressed.get(), block->uncompressedSize, deps::oodle::OODLE_FS_YES) }; + + if (ret != block->uncompressedSize) { + throw std::runtime_error(std::format("Can't decompress block, returned size isn't the expected one: 0x{:x} != 0x{:x}", ret, block->uncompressedSize)); + } + } + break; + default: + throw std::runtime_error(std::format("No fastfile decompressor for type {}", (int)buffer->compression)); + } + + + utils::WriteValue(ffdata, decompressed.get(), block->uncompressedSize); } - LOG_INFO("server ..... {}", (buffer->server ? "true" : "false")); - LOG_INFO("encrypted .. {}", (buffer->encrypted ? "true" : "false")); - LOG_INFO("platform ... {}", (int)buffer->platform); - LOG_INFO("builder .... {}", buffer->builder); - LOG_INFO("timestamp .. {}", buffer->timestamp); - + LOG_INFO("Decompressed 0x{:x} byte(s)", ffdata.size()); - for (size_t i = 0; i < 4; i++) { - LOG_INFO("archiveChecksum[{}] = {:x}", i, buffer->archiveChecksum[i]); + if (!utils::WriteFile(deco, ffdata)) { + LOG_ERROR("Can't write to {}", deco.string()); + return tool::BASIC_ERROR; } - LOG_INFO("key ........ {}", buffer->key); - LOG_INFO("compress ... {}", (int)buffer->compression); - LOG_INFO("size ....... {}B", utils::FancyNumber(fileBuff.size())); + core::bytebuffer::ByteBuffer buff{ ffdata }; - // 6 = inflate? - // 7 = lzma? - // 8 = none? + uint64_t magic{ 0x00a0d43534780 }; + uint64_t magicMask{ 0xFFFFFFFFFFFFF }; + size_t loc{}; + std::filesystem::path out{ "scriptparsetree_t8ff" }; - if (buffer->encrypted) { - auto checksum32 = fastfiles::ComputeChecksum32(buffer->key, (unsigned int)strlen(buffer->key), 0); - LOG_INFO("checksum32 . {}", checksum32); - } + byte* start{ buff.Ptr() }; + while (true) { + loc = buff.FindMasked((byte*)&magic, (byte*)&magicMask, sizeof(magic)); + if (loc == std::string::npos) break; + struct T8GSCOBJ { + byte magic[8]; + int32_t crc; + int32_t pad; + uint64_t name; + }; - if (!tool::NotEnoughParam(argc, 2)) { - utils::OutFileCE os{ argv[3] }; + buff.Goto(loc); - if (!os) { - LOG_ERROR("Can't open {}", argv[3]); - return tool::BASIC_ERROR; + if (!buff.CanRead(sizeof(T8GSCOBJ))) { + break; // can't read buffer + } + + // we are 32 bytes aligned + T8GSCOBJ* obj{ buff.Ptr() }; + + uint64_t name{ obj->name }; + + + byte* sptCan{ buff.Ptr() }; + + while (*(uint64_t*)sptCan != name) { + sptCan--; + if (start == sptCan) { + break; + } + } + if (start == sptCan) { + loc++; + continue; + } + + + uint64_t smagic{ *reinterpret_cast(obj) }; + + size_t size; + struct T8SPT { + uint64_t name; + uint64_t pad0; + uintptr_t buffer; + uint32_t size; + uint32_t pad02; + }; + struct T8SPT35 { + uint64_t name; + uintptr_t buffer; + uint32_t size; + uint32_t pad02; + }; + + if (smagic == 0x36000a0d43534780) { + size = ((T8SPT*)sptCan)->size; + } else if (smagic == 0x35000a0d43534780) { + size = ((T8SPT35*)sptCan)->size; + } + else { + LOG_ERROR("Invalid magic 0x{:x}", smagic); + loc++; + continue; } - WriteHex(os, 0, reinterpret_cast(buffer), sizeof(*buffer)); + LOG_TRACE("gsc: 0x{:x} 0x{:x} 0x{:x}: {}", smagic, loc, size, hashutils::ExtractTmpScript(obj->name)); - os << "-----------------------------\n"; - WriteHex(os, reinterpret_cast(buffer + 1) - reinterpret_cast(buffer), reinterpret_cast(buffer + 1), fileBuff.size() - sizeof(*buffer)); + if (!buff.CanRead(size)) { + loc++; + LOG_ERROR("Bad size 0x{:x}", size); + continue; + } + buff.Skip(size); + + std::filesystem::path outFile{ out / std::format("vm_{:x}/script_{:x}.gscc", smagic, obj->name) }; + std::filesystem::create_directories(outFile.parent_path()); + + if (!utils::WriteFile(outFile, obj->magic, size)) { + LOG_ERROR("Can't write {}", outFile.string()); + } + else { + LOG_INFO("Dump {}", outFile.string()); + } - LOG_INFO("Read into {}", argv[3]); + loc++; } + + + LOG_INFO("Done"); + return tool::OK; } + int ffdaes(Process& proc, int argc, const char* argv[]) { aesKey_t keys[64]; @@ -359,7 +592,7 @@ namespace { #ifndef CI_BUILD -ADD_TOOL(fftest, "fastfile", " [ff]", "test fastfile", nullptr, fftest); +ADD_TOOL(fftest, "fastfile", " [ff]", "test fastfile", fftest); ADD_TOOL(casctest, "fastfile", " [path]", "test casc", nullptr, casctest); ADD_TOOL(h32ck, "fastfile", " [path]", "hash checksum", nullptr, h32ck); ADD_TOOL(daes, "bo4", "", "dump AES keys", L"BlackOps4.exe", ffdaes); diff --git a/src/acts/tools/test.cpp b/src/acts/tools/test.cpp index bc9fa6e..079e708 100644 --- a/src/acts/tools/test.cpp +++ b/src/acts/tools/test.cpp @@ -96,6 +96,22 @@ namespace { return tool::OK; } + int oodlecomp(int argc, const char* argv[]) { + if (tool::NotEnoughParam(argc, 1)) return tool::BAD_USAGE; + + deps::oodle::Oodle oodle{ deps::oodle::OO2CORE_6 }; + + byte rawData[0x1000]; + + for (size_t i = 0; i < sizeof(rawData); i++) { + rawData[i] = (byte)(((int)rawData[i - 1] ^ (int)i) * 0x11); + } + + auto ptr{ std::make_unique((size_t)oodle.GetCompressedBufferSizeNeeded(deps::oodle::OODLE_COMP_KRAKEN, sizeof(rawData))) }; + + //oodle.Compress(deps::oodle::OODLE_COMP_KRAKEN, &rawData[0], sizeof(rawData)); + + } int oodleload(int argc, const char* argv[]) { if (tool::NotEnoughParam(argc, 1)) return tool::BAD_USAGE; @@ -126,4 +142,5 @@ ADD_TOOL(memalloctest, "dev", "", "Tests", nullptr, memalloctest); ADD_TOOL(wget, "dev", " [url]", "Tests", nullptr, testurl); ADD_TOOL(cfgtest, "dev", "", "", nullptr, cfgtest); ADD_TOOL(oodleload, "dev", " [exe]", "", oodleload); +//ADD_TOOL(oodlecomp, "dev", "", "", oodlecomp); #endif \ No newline at end of file diff --git a/src/shared/deps/oodle.hpp b/src/shared/deps/oodle.hpp index d2a2205..f19379e 100644 --- a/src/shared/deps/oodle.hpp +++ b/src/shared/deps/oodle.hpp @@ -7,13 +7,84 @@ namespace deps::oodle { constexpr const char* OO2CORE_7 = "oo2core_7_win64"; constexpr const char* OO2CORE_8 = "oo2core_8_win64"; - typedef size_t(*POodleLZ_Decompress)(void* src, size_t srcLen, void* dest, size_t destLen, int a5, int a6, int a7, __int64 a8, __int64 a9, unsigned int(*a10)(__int64, __int64, __int64, __int64), __int64 a11, __int64 a12, __int64 a13, int a14); + enum OodleFuzeSafe : int { + OODLE_FS_NO = 0, + OODLE_FS_YES = 1 + }; + + enum OodleCheckCrcValues : int { + OODLE_CRC_NO = 0, + OODLE_CRC_YES = 1 + }; + + enum OodleVerbosity : int { + OODLE_VERB_NONE = 0, + OODLE_VERB_MIN = 1, + OODLE_VERB_SOME = 2, + OODLE_VERB_LOTS = 3 + }; + + enum OodleThreadPhase : int { + OODLE_TP_ThreadPhase1 = 1, + OODLE_TP_ThreadPhase2 = 2, + OODLE_TP_ThreadPhaseAll = 3, + OODLE_TP_Unthreaded = OODLE_TP_ThreadPhaseAll + }; + + enum OodleCompressor : int { + OODLE_COMP_NONE = 3, + OODLE_COMP_KRAKEN = 8, + OODLE_COMP_LEVIATHAN = 13, + OODLE_COMP_MERMAID = 9, + OODLE_COMP_SELKIE = 11, + OODLE_COMP_HYDRA = 12, + }; + enum OodleCompressionLevel : int { + OODLE_COMPL_NONE = 0, + OODLE_COMPL_SUPERFAST = 1, + OODLE_COMPL_NORMAL = 4, + OODLE_COMPL_OPTIMAL5 = 9, + }; + + typedef int(*POodleLZ_Compress)( + OodleCompressor compressor, + const void* src, + int32_t srcLen, + void* dest, + OodleCompressionLevel level, + const void* pOptions, + const void* dictionaryBase, + const void* lrm, + void* scratchMem, + int32_t scratchSize + ); + + typedef int32_t(*POodleLZ_Decompress)( + void* src, + uint32_t srcLen, + void* dest, + uint32_t destLen, + OodleFuzeSafe fuzzSafe, + OodleCheckCrcValues checkCrc, + OodleVerbosity verbosity, + byte* decBufBase, + uint64_t decBufSize, + uint64_t fpCallback, + void* callbackUserData, + byte* decoderMemory, + uint64_t decoderMemorySize, + OodleThreadPhase threadPhase); typedef void(*POodle_GetConfigValues)(int32_t* cfg); + typedef int32_t(*POodleLZ_GetCompressedBufferSizeNeeded)(OodleCompressor compressor, int32_t rawSize); + + class Oodle { hook::library::Library oodle{}; POodleLZ_Decompress OodleLZ_Decompress{}; POodle_GetConfigValues Oodle_GetConfigValues{}; + POodleLZ_GetCompressedBufferSizeNeeded OodleLZ_GetCompressedBufferSizeNeeded{}; + POodleLZ_Compress OodleLZ_Compress{}; public: Oodle() {} @@ -46,6 +117,11 @@ namespace deps::oodle { LOG_TRACE("Loaded oodle version 0x{:x}", GetVersion()); } + if ((OodleLZ_GetCompressedBufferSizeNeeded = oodle.GetProc("OodleLZ_GetCompressedBufferSizeNeeded")) + && (OodleLZ_Compress = oodle.GetProc("OodleLZ_Compress"))) { + LOG_TRACE("Loaded oodle compress"); + } + return true; } @@ -75,6 +151,20 @@ namespace deps::oodle { Oodle_GetConfigValues(cfg); } + void Compress(OodleCompressor compressor, const void* src, int32_t srcLen, void* dest, OodleCompressionLevel level = OODLE_COMPL_NORMAL) const { + if (!OodleLZ_Compress) { + throw std::runtime_error("OodleLZ_Compress not available or oodle not loaded"); + } + OodleLZ_Compress(compressor, src, srcLen, dest, level, nullptr, nullptr, nullptr, nullptr, 0); + } + + int32_t GetCompressedBufferSizeNeeded(OodleCompressor compressor, int32_t rawSize) const { + if (!OodleLZ_GetCompressedBufferSizeNeeded) { + throw std::runtime_error("OodleLZ_GetCompressedBufferSizeNeeded not available or oodle not loaded"); + } + return OodleLZ_GetCompressedBufferSizeNeeded(compressor, rawSize); + } + int32_t GetVersion() const { int32_t cfg[7]; GetConfigValues(cfg); @@ -85,12 +175,15 @@ namespace deps::oodle { return oodle; } - size_t Decompress(void* src, size_t srcLen, void* dest, size_t destLen) const { + int Decompress( + void* src, uint32_t srcLen, void* dest, uint32_t destLen, + OodleFuzeSafe fuzeSafe = OODLE_FS_NO, OodleCheckCrcValues checkCrc = OODLE_CRC_NO, OodleVerbosity verbosity = OODLE_VERB_NONE, + OodleThreadPhase threadPhase = OODLE_TP_Unthreaded) const { if (!OodleLZ_Decompress) { throw std::runtime_error("Oodle not loaded"); } - return OodleLZ_Decompress(src, srcLen, dest, destLen, 0, 0, 0, 0, 0, nullptr, 0, 0, 0, 0); + return OodleLZ_Decompress(src, srcLen, dest, destLen, fuzeSafe, checkCrc, verbosity, nullptr, 0, 0, nullptr, nullptr, 0, threadPhase); } }; } \ No newline at end of file