Skip to content

Commit

Permalink
add localized entries, spt dbg source and fix oodle67 compression
Browse files Browse the repository at this point in the history
  • Loading branch information
@ate47
ate47 committed Feb 16, 2025
1 parent 582aa20 commit d359216
Show file tree
Hide file tree
Showing 8 changed files with 173 additions and 13 deletions.
7 changes: 6 additions & 1 deletion src/acts/tools/ff/compressors/compressor_bo4.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,9 @@ namespace {

}
alg = fastfile::GetFastFileCompressionAlgorithm(compression);
if (opt.useHC) {
alg = alg | utils::compress::COMP_HIGH_COMPRESSION;
}
constexpr size_t maxSize{ utils::GetMaxSize<int32_t>() };
if (opt.chunkSize > maxSize) {
LOG_WARNING("Chunk size can't be above {}", maxSize);
Expand All @@ -65,6 +68,7 @@ namespace {

size_t idx{};
std::vector<byte> compressBuffer{};
size_t compressedSize{};
LOG_TRACE("start compressing chunk 0x{:x} byte(s) using {}...", remainingSize, alg);
while (remainingSize > 0) {
uint32_t uncompressedSize{ (uint32_t)std::min<size_t>(chunkSize, remainingSize) };
Expand All @@ -74,6 +78,7 @@ namespace {
throw std::runtime_error(std::format("Can't compress chunk 0x{:x} of size 0x{:x}", idx - 1, uncompressedSize));
}
uint32_t alignedSize{ utils::Aligned<uint32_t>((uint32_t)compressBuffer.size()) };
compressedSize += compressBuffer.size();

uint32_t blockOffset{ (uint32_t)utils::Allocate(out, sizeof(fastfile::DBStreamHeader) + alignedSize) };

Expand Down Expand Up @@ -139,7 +144,7 @@ namespace {
LOG_WARNING(".fd file generator not implemented");
}

LOG_INFO("Compressed {} into {}", ctx.ffname, outputFile.string());
LOG_INFO("Compressed {} into {} ({} -> {} bytes / {}% saved)", ctx.ffname, outputFile.string(), ctx.data.size(), compressedSize, (100 - 100 * compressedSize / ctx.data.size()));
}

};
Expand Down
4 changes: 4 additions & 0 deletions src/acts/tools/ff/fastfile_handlers.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -294,6 +294,9 @@ namespace fastfile {
else if (!_strcmpi("--server", arg)) {
server = true;
}
else if (!strcmp("-H", arg) || !_strcmpi("--high-compression", arg)) {
useHC = true;
}
else if (!_strcmpi("--chunkSize", arg)) {
if (i + 1 == endIndex) {
std::cerr << "Missing value for param: " << arg << "!\n";
Expand Down Expand Up @@ -325,6 +328,7 @@ namespace fastfile {
LOG_INFO("-c --compressor : Compressor to use");
LOG_INFO("-C --compressors : Print compressors");
LOG_INFO("--chunkSize [size] : Set chunk sizes");
LOG_INFO("-H --high-compression : Use high compression (if available)");
}


Expand Down
1 change: 1 addition & 0 deletions src/acts/tools/ff/fastfile_handlers.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,7 @@ namespace fastfile {
bool m_header{};
bool printCompressors{};
bool printLinkers{};
bool useHC{};
size_t chunkSize{};
std::filesystem::path m_output{ "output_ff" };
const char* ffname{};
Expand Down
39 changes: 39 additions & 0 deletions src/acts/tools/ff/linkers/bo4/linker_bo4_localize.cpp
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
#include <includes.hpp>
#include <tools/ff/linkers/linker_bo4.hpp>

namespace fastfile::linker::bo4 {
class LocalizeWorker : public LinkerWorker {
public:
LocalizeWorker() : LinkerWorker("Localize") {}

void Compute(BO4LinkContext& ctx) override {

std::filesystem::path localizePath{ ctx.linkCtx.input / "localize.json" };
core::config::Config localize{ localizePath };
localize.SyncConfig(false);
if (localize.main.Empty()) return; // nothing to force

struct LocalizeEntry {
uintptr_t value; // const char*
XHash name;
};

size_t added{};
for (auto& [k, v] : localize.main.GetObj()) {
if (!k.IsString() || !v.IsString()) {
LOG_WARNING("Invalid localize: {} -> {}", k.GetString(), v.GetString());
continue;
}
LocalizeEntry& header{ utils::Allocate<LocalizeEntry>(ctx.assetData) };
header.name.hash = hash::Hash64Pattern(k.GetString());
header.value = fastfile::ALLOC_PTR;
utils::WriteString(ctx.assetData, v.GetString());
ctx.assets.emplace_back(games::bo4::pool::ASSET_TYPE_LOCALIZE_ENTRY, fastfile::ALLOC_PTR);
added++;
}
LOG_INFO("Added asset localizeentry {}: {} entry(ies)", localizePath.string(), added);
}
};

utils::ArrayAdder<LocalizeWorker, LinkerWorker> impl{ GetWorkers() };
}
63 changes: 59 additions & 4 deletions src/acts/tools/ff/linkers/bo4/linker_bo4_scriptparsetree.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ namespace fastfile::linker::bo4 {
uintptr_t buffer;
uint32_t len;
}; static_assert(sizeof(ScriptParseTree) == 0x20);

ScriptParseTree& spt{ utils::Allocate<ScriptParseTree>(ctx.assetData) };

spt.name.hash = obj.name;
Expand All @@ -34,11 +34,56 @@ namespace fastfile::linker::bo4 {
LOG_INFO("Added asset scriptparsetree {} (hash_{:x})", path.string(), obj.name);
}

static tool::gsc::opcode::Platform GetGSCPlatform(fastfile::FastFilePlatform plt) {
switch (plt) {
case fastfile::XFILE_PC: return tool::gsc::opcode::PLATFORM_PC;
case fastfile::XFILE_PLAYSTATION: return tool::gsc::opcode::PLATFORM_PLAYSTATION;
case fastfile::XFILE_XBOX: return tool::gsc::opcode::PLATFORM_XBOX;
case fastfile::XFILE_DEV: return tool::gsc::opcode::PLATFORM_PC_ALPHA;
default: throw std::runtime_error("Invalid pltform for this ff");
}
}

static void AddGscDBGHeader(BO4LinkContext& ctx, std::vector<byte>& buffer, std::string& preproc, const std::filesystem::path& path) {
if (buffer.size() < sizeof(tool::gsc::T8GSCOBJ)) {
LOG_ERROR("Can't read compiled gsc header for {}", path.string());
return;
}
tool::gsc::T8GSCOBJ& obj{ *(tool::gsc::T8GSCOBJ*)buffer.data() };
struct ScriptParseTreeDBG
{
XHash name;
int gdbLen;
int srcLen;
uintptr_t gdb;
uintptr_t src;
}; static_assert(sizeof(ScriptParseTreeDBG) == 0x28);

ScriptParseTreeDBG& spt{ utils::Allocate<ScriptParseTreeDBG>(ctx.assetData) };

spt.name.hash = obj.name;
spt.src = fastfile::ALLOC_PTR;
spt.srcLen = (uint32_t)preproc.size();
// todo: implement gdb compiler

// write script header
preproc.push_back(0); // the game is reading (len + 1) so we add a byte at the end
utils::WriteValue(ctx.assetData, preproc.data(), preproc.size());

ctx.assets.emplace_back(games::bo4::pool::ASSET_TYPE_SCRIPTPARSETREEDBG, fastfile::ALLOC_PTR);
LOG_INFO("Added asset scriptparsetreedbg {} (hash_{:x})", path.string(), obj.name);
}

void Compute(BO4LinkContext& ctx) override {
std::vector<std::filesystem::path> scripts{};
std::filesystem::path scriptDir{ ctx.linkCtx.input / "scripts" };
utils::GetFileRecurseExt(scriptDir, scripts, ".csc\0.gsc\0", true);

bool cfgGenDBG{ ctx.ffConfig.GetBool("gsc.gendbg", false) };
bool cfgDev{ ctx.ffConfig.GetBool("gsc.dev", false) };
bool cfgNoDevCallInline{ ctx.ffConfig.GetBool("gsc.noDevCallInline", false) };
bool genDevBlockAsComment{ ctx.ffConfig.GetBool("gsc.devBlockAsComment", false) };

for (const std::filesystem::path& sub : scripts) {
std::filesystem::path path{ scriptDir / sub };
std::filesystem::path scriptName{ std::filesystem::path{"scripts"} / sub };
Expand All @@ -48,9 +93,17 @@ namespace fastfile::linker::bo4 {
acts::compiler::CompilerConfig cfg{};
std::string snp{ scriptName.string() };
cfg.name = hashutils::CleanPath(snp.data());
cfg.platform = tool::gsc::opcode::PLATFORM_PC;
cfg.vm = tool::gsc::opcode::VMI_T8;
cfg.platform = GetGSCPlatform(ctx.linkCtx.opt.platform);
cfg.vm = tool::gsc::opcode::VMI_T8; // read cfg?
cfg.detourType = acts::compiler::DETOUR_ACTS;
cfg.computeDevOption = cfgDev;
cfg.computeDevOption = cfgDev;
std::string preprocOutput{};
if (cfgGenDBG) {
cfg.preprocOutput = &preprocOutput;
}
cfg.noDevCallInline = cfgNoDevCallInline;
cfg.processorOpt.devBlockAsComment = genDevBlockAsComment;
cfg.clientScript = scriptName.extension() == ".csc";
cfg.processorOpt.defines.insert(std::format("_FF_GEN_{}", ctx.linkCtx.ffname));

Expand All @@ -67,7 +120,9 @@ namespace fastfile::linker::bo4 {

LOG_INFO("Compiled {} ({})", path.string(), cfg.name);
AddGscHeader(ctx, buffer, path);
// TODO: generate and add ScriptParseTreeDBG header
if (cfgGenDBG) {
AddGscDBGHeader(ctx, buffer, preprocOutput, path);
}
}

scripts.clear();
Expand Down
3 changes: 2 additions & 1 deletion src/acts/tools/utils/compress_utils.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,8 @@ namespace utils::compress {
}

bool CompressBuffer(CompressionAlgorithm alg, const void* src, size_t srcSize, std::vector<byte>& out) {
out.resize(GetCompressSize(alg, srcSize));
size_t allocSize{ GetCompressSize(alg, srcSize) };
out.resize(allocSize);

size_t outLen{ out.size() };
if (!Compress(alg, out.data(), &outLen, src, srcSize)) {
Expand Down
44 changes: 40 additions & 4 deletions src/shared/deps/oodle.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,18 @@ namespace deps::oodle {
Oodle::Oodle(const char* libname) {
LoadOodle(libname);
}
namespace {
void DefaultPrintF(int unk, const char* filename, int len, const char* fmt, ...) {
if (HAS_LOG_LEVEL(core::logs::LVL_INFO)) {
va_list va{};
va_start(va, fmt);

core::logs::log(core::logs::LVL_INFO, filename, len, utils::vap(fmt, va));

va_end(va);
}
}
}

bool Oodle::LoadOodle(const char* libname) {
FreeOodle();
Expand All @@ -39,9 +51,15 @@ namespace deps::oodle {
LOG_TRACE("Loaded oodle version 0x{:x}", GetVersion());
}

if ((OodleLZ_GetCompressedBufferSizeNeeded = oodle.GetProc<POodleLZ_GetCompressedBufferSizeNeeded>("OodleLZ_GetCompressedBufferSizeNeeded"))
if ((OodleLZ_GetCompressedBufferSizeNeeded.v8 = oodle.GetProc<POodleLZ_GetCompressedBufferSizeNeededV8>("OodleLZ_GetCompressedBufferSizeNeeded"))
&& (OodleLZ_Compress = oodle.GetProc<POodleLZ_Compress>("OodleLZ_Compress"))) {
LOG_TRACE("Loaded oodle compress");

useV8 = OodleLZ_GetCompressedBufferSizeNeeded.v8(deps::oodle::OODLE_COMP_KRAKEN, 10000) > 10000;
}

if (OodleCore_Plugins_SetPrintf = oodle.GetProc<POodleCore_Plugins_SetPrintf>("OodleCore_Plugins_SetPrintf")) {
OodleCore_Plugins_SetPrintf(DefaultPrintF);
}

return true;
Expand All @@ -65,9 +83,15 @@ namespace deps::oodle {
LOG_TRACE("Loaded oodle version 0x{:x}", GetVersion());
}

if ((OodleLZ_GetCompressedBufferSizeNeeded = oodle.GetProc<POodleLZ_GetCompressedBufferSizeNeeded>("OodleLZ_GetCompressedBufferSizeNeeded"))
if ((OodleLZ_GetCompressedBufferSizeNeeded.v8 = oodle.GetProc<POodleLZ_GetCompressedBufferSizeNeededV8>("OodleLZ_GetCompressedBufferSizeNeeded"))
&& (OodleLZ_Compress = oodle.GetProc<POodleLZ_Compress>("OodleLZ_Compress"))) {
LOG_TRACE("Loaded oodle compress");

useV8 = OodleLZ_GetCompressedBufferSizeNeeded.v8(deps::oodle::OODLE_COMP_KRAKEN, 10000) >= 10000;
}

if (OodleCore_Plugins_SetPrintf = oodle.GetProc<POodleCore_Plugins_SetPrintf>("OodleCore_Plugins_SetPrintf")) {
OodleCore_Plugins_SetPrintf(DefaultPrintF);
}

return true;
Expand Down Expand Up @@ -109,10 +133,15 @@ namespace deps::oodle {
}

int32_t Oodle::GetCompressedBufferSizeNeeded(OodleCompressor compressor, int32_t rawSize) const {
if (!OodleLZ_GetCompressedBufferSizeNeeded) {
if (!OodleLZ_GetCompressedBufferSizeNeeded.v8) {
throw std::runtime_error("OodleLZ_GetCompressedBufferSizeNeeded not available or oodle not loaded");
}
return OodleLZ_GetCompressedBufferSizeNeeded(compressor, rawSize);
if (useV8) {
return OodleLZ_GetCompressedBufferSizeNeeded.v8(compressor, rawSize);
}
else {
return OodleLZ_GetCompressedBufferSizeNeeded.v67(rawSize);
}
}

int32_t Oodle::GetVersion() const {
Expand All @@ -121,6 +150,13 @@ namespace deps::oodle {
return cfg[6];
}

void Oodle::SetPrintf(printffunc func) const {
if (!OodleCore_Plugins_SetPrintf) {
throw std::runtime_error("OodleCore_Plugins_SetPrintf not available or oodle not loaded");
}
OodleCore_Plugins_SetPrintf(func);
}

int Oodle::Decompress(
const void* src, uint32_t srcLen, void* dest, uint32_t destLen, OodleFuzeSafe fuzeSafe, OodleCheckCrcValues checkCrc, OodleVerbosity verbosity, OodleThreadPhase threadPhase) const {
if (!OodleLZ_Decompress) {
Expand Down
25 changes: 22 additions & 3 deletions src/shared/deps/oodle.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,12 +33,20 @@ namespace deps::oodle {
};

enum OodleCompressor : int {
OODLE_COMP_LZH = 0,
OODLE_COMP_LZH_LW = 1,
OODLE_COMP_LZNIB = 2,
OODLE_COMP_NONE = 3,
OODLE_COMP_LZB16 = 4,
OODLE_COMP_LZBW = 5,
OODLE_COMP_LZA = 6,
OODLE_COMP_LZNA = 7,
OODLE_COMP_KRAKEN = 8,
OODLE_COMP_LEVIATHAN = 13,
OODLE_COMP_MERMAID = 9,
OODLE_COMP_BITKNIT = 10,
OODLE_COMP_SELKIE = 11,
OODLE_COMP_HYDRA = 12,
OODLE_COMP_LEVIATHAN = 13,
};
enum OodleCompressionLevel : int {
OODLE_COMPL_NONE = 0,
Expand Down Expand Up @@ -77,15 +85,24 @@ namespace deps::oodle {
OodleThreadPhase threadPhase);
typedef void(*POodle_GetConfigValues)(int32_t* cfg);

typedef int32_t(*POodleLZ_GetCompressedBufferSizeNeeded)(OodleCompressor compressor, int32_t rawSize);

typedef int32_t(*POodleLZ_GetCompressedBufferSizeNeededV8)(OodleCompressor compressor, int32_t rawSize);
typedef int32_t(*POodleLZ_GetCompressedBufferSizeNeededV67)(int32_t rawSize);
union POodleLZ_GetCompressedBufferSizeNeeded {
POodleLZ_GetCompressedBufferSizeNeededV8 v8;
POodleLZ_GetCompressedBufferSizeNeededV67 v67;
};
typedef void (*printffunc)(int a1, const char* filename, int line, const char* format, ...);
typedef void(*POodleCore_Plugins_SetPrintf)(printffunc func);


class Oodle {
hook::library::Library oodle{ (HMODULE)0 };
POodleLZ_Decompress OodleLZ_Decompress{};
POodle_GetConfigValues Oodle_GetConfigValues{};
POodleLZ_GetCompressedBufferSizeNeeded OodleLZ_GetCompressedBufferSizeNeeded{};
bool useV8{};
POodleLZ_Compress OodleLZ_Compress{};
POodleCore_Plugins_SetPrintf OodleCore_Plugins_SetPrintf{};

public:
Oodle() {}
Expand Down Expand Up @@ -115,6 +132,8 @@ namespace deps::oodle {

int32_t GetVersion() const;

void SetPrintf(printffunc func) const;

int Decompress(
const void* src, uint32_t srcLen, void* dest, uint32_t destLen,
OodleFuzeSafe fuzeSafe = OODLE_FS_NO, OodleCheckCrcValues checkCrc = OODLE_CRC_NO, OodleVerbosity verbosity = OODLE_VERB_NONE,
Expand Down

0 comments on commit d359216

Please sign in to comment.