bo4 fast file linker (spt/rawfile)

src/acts/compiler/gsc_compiler.cpp

@@ -6163,10 +6163,8 @@ namespace acts::compiler {
         VmInfo* vmInfo{ config.GetVm() };
         InputInfo info{};
         for (const std::filesystem::path& file : files) {
-            auto ext = file.extension();
-
             if (!info.container.AppendFile(file)) {
-                throw std::runtime_error(std::format("Can't read file {}", file.string()));
+                throw std::runtime_error(std::format("Can't read file '{}'", file.string()));
             }
         }
 

src/acts/compiler/gsc_compiler.hpp

@@ -37,7 +37,7 @@ namespace acts::compiler {
 
 	void CompileGsc(const std::vector<std::filesystem::path>& files, std::vector<byte>& data, CompilerConfig& cfg);
 	inline void CompileGsc(const std::filesystem::path& file, std::vector<byte>& data, CompilerConfig& cfg) {
-		std::vector<std::filesystem::path> files{ 1 };
+		std::vector<std::filesystem::path> files{};
 		files.emplace_back(file);
 		CompileGsc(files, data, cfg);
 	}

src/acts/tools/ff/compressors/compressor_bo4.cpp

@@ -104,10 +104,10 @@ namespace {
 			// write header data
 			XFile& header{ *(XFile*)out.data() };
 			*(uint64_t*)&header.magic[0] = 0x3030303066664154;
+			header.version = 0x27F;
 			header.platform = ctx.opt.platform;
 			header.server = ctx.opt.server;
-			header.timestamp = utils::GetTimestamp();
-			header.version = 0x27F;
+			header.timestamp = utils::GetTimestamp() / 1000;
 			header.encrypted = false;
 			header.size = ctx.data.size();
 			header.compression = compression;

src/acts/tools/ff/fastfile_handlers.hpp

@@ -9,6 +9,8 @@ namespace fastfile {
 	inline bool IsSame(T t, int64_t val) {
 		return reinterpret_cast<T>(val) == t;
 	}
+
+	constexpr uintptr_t ALLOC_PTR = static_cast<uintptr_t>(-1);
 
 	struct DBStreamHeader {
 		uint32_t compressedSize;
@@ -172,8 +174,8 @@ namespace fastfile {
 		const char* ffname{};
 		core::memory_allocator::MemoryAllocator strs{};
 
-		FastFileLinkerContext(FastFileLinkerOption& opt, std::filesystem::path input, std::vector<byte>& data)
-			: opt(opt), input(input), data(data) {
+		FastFileLinkerContext(FastFileLinkerOption& opt, std::filesystem::path in, std::vector<byte>& data)
+			: opt(opt), input(std::filesystem::absolute(in)), data(data) {
 			if (opt.ffname) ffname = opt.ffname;
 			else {
 				inputFileNameStr = input.filename().string();

src/acts/tools/ff/linkers/bo4/linker_bo4_rawfile.cpp

@@ -0,0 +1,52 @@
+#include <includes.hpp>
+#include <tools/ff/linkers/linker_bo4.hpp>
+
+namespace fastfile::linker::bo4 {
+	class RawFileWorker : public LinkerWorker {
+	public:
+		RawFileWorker() : LinkerWorker("RawFile") {}
+
+		void Compute(BO4LinkContext& ctx) override {
+			std::vector<std::filesystem::path> files{};
+			utils::GetFileRecurseExt(ctx.linkCtx.input, files,
+				// all the extensions to read into the rawfile assets
+				".cfg\0"
+				".txt\0"
+				".vision\0"
+				".graph\0"
+				".baseline\0"
+				// extension used in the dump
+				".raw\0" 
+				, true);
+
+			for (const std::filesystem::path& rfpath : files) {
+				std::filesystem::path path{ ctx.linkCtx.input / rfpath };
+				std::string buffer{};
+				if (!utils::ReadFile(path, buffer)) {
+					LOG_ERROR("Can't read {}", path.string());
+					ctx.error = true;
+					continue;
+				}
+
+				struct RawFile {
+					XHash name;
+					int32_t len;
+					uintptr_t buffer;
+				}; static_assert(sizeof(RawFile) == 0x20);
+
+				RawFile& rf{ utils::Allocate<RawFile>(ctx.assetData) };
+
+				rf.name.hash = HashPathName(rfpath);
+				rf.buffer = fastfile::ALLOC_PTR;
+				rf.len = (uint32_t)buffer.size();
+
+				// write header
+				utils::WriteValue(ctx.assetData, buffer.data(), buffer.length() + 1); // add \0
+				ctx.assets.emplace_back(games::bo4::pool::ASSET_TYPE_RAWFILE, fastfile::ALLOC_PTR);
+				LOG_INFO("Added asset rawfile {} (hash_{:x})", path.string(), rf.name.hash);
+			}
+		}
+	};
+
+	utils::ArrayAdder<RawFileWorker, LinkerWorker> impl{ GetWorkers() };
+}

src/acts/tools/ff/linkers/bo4/linker_bo4_scriptparsetree.cpp

@@ -0,0 +1,88 @@
+#include <includes.hpp>
+#include <tools/ff/linkers/linker_bo4.hpp>
+#include <tools/gsc.hpp>
+#include <compiler/gsc_compiler.hpp>
+
+namespace fastfile::linker::bo4 {
+	class ScriptParseTreeWorker : public LinkerWorker {
+	public:
+		ScriptParseTreeWorker() : LinkerWorker("ScriptParseTree") {}
+
+		static void AddGscHeader(BO4LinkContext& ctx, std::vector<byte>& buffer, const std::filesystem::path& path) {
+			if (buffer.size() < sizeof(tool::gsc::T8GSCOBJ)) {
+				LOG_ERROR("Can't read compiled gsc header for {}", path.string());
+				return;
+			}
+			tool::gsc::T8GSCOBJ& obj{ *(tool::gsc::T8GSCOBJ*)buffer.data() };
+
+			struct ScriptParseTree {
+				XHash name;
+				uintptr_t buffer;
+				uint32_t len;
+			}; static_assert(sizeof(ScriptParseTree) == 0x20);
+
+			ScriptParseTree& spt{ utils::Allocate<ScriptParseTree>(ctx.assetData) };
+
+			spt.name.hash = obj.name;
+			spt.buffer = fastfile::ALLOC_PTR;
+			spt.len = (uint32_t)buffer.size();
+
+			// write script header
+			buffer.push_back(0); // the game is reading (len + 1) so we add a byte at the end
+			utils::WriteValue(ctx.assetData, buffer.data(), buffer.size());
+			ctx.assets.emplace_back(games::bo4::pool::ASSET_TYPE_SCRIPTPARSETREE, fastfile::ALLOC_PTR);
+			LOG_INFO("Added asset scriptparsetree {} (hash_{:x})", path.string(), spt.name.hash);
+		}
+
+		void Compute(BO4LinkContext& ctx) override {
+			std::vector<std::filesystem::path> scripts{};
+			std::filesystem::path scriptDir{ ctx.linkCtx.input / "scripts" };
+			utils::GetFileRecurseExt(scriptDir, scripts, ".csc\0.gsc\0", true);
+
+			for (const std::filesystem::path& sub : scripts) {
+				std::filesystem::path path{ scriptDir / sub };
+				std::filesystem::path scriptName{ std::filesystem::path{"scripts"} / sub };
+				LOG_TRACE("Processing {} ({})", scriptName.string(), path.string());
+
+				// compile file
+				acts::compiler::CompilerConfig cfg{};
+				std::string snp{ scriptName.string() };
+				cfg.name = hashutils::CleanPath(snp.data());
+				cfg.platform = tool::gsc::opcode::PLATFORM_PC;
+				cfg.vm = tool::gsc::opcode::VMI_T8;
+				cfg.detourType = acts::compiler::DETOUR_ACTS;
+				cfg.clientScript = scriptName.extension() == ".csc";
+				cfg.processorOpt.defines.insert(std::format("_FF_GEN_{}", ctx.linkCtx.ffname));
+
+				std::vector<byte> buffer{};
+				try {
+					acts::compiler::CompileGsc(path, buffer, cfg);
+				}
+				catch (std::runtime_error& re) {
+					LOG_ERROR("Can't compile {}: {}", path.string(), re.what());
+					ctx.error = true;
+					continue;
+				}
+
+
+				LOG_INFO("Compiled {} ({})", path.string(), cfg.name);
+				AddGscHeader(ctx, buffer, path);
+				// TODO: generate and add ScriptParseTreeDBG header
+			}
+
+			scripts.clear();
+			utils::GetFileRecurseExt(ctx.linkCtx.input, scripts, ".cscc\0.gscc\0");
+			for (const std::filesystem::path& path : scripts) {
+				std::vector<byte> buffer{};
+				if (!utils::ReadFile(path, buffer)) {
+					LOG_ERROR("Can't read {}", path.string());
+					ctx.error = true;
+					continue;
+				}
+				AddGscHeader(ctx, buffer, path);
+			}
+		}
+	};
+
+	utils::ArrayAdder<ScriptParseTreeWorker, LinkerWorker> impl{ GetWorkers() };
+}

src/acts/tools/ff/linkers/linker_bo4.cpp

@@ -1,24 +1,27 @@
 #include <includes.hpp>
-#include <core/config.hpp>
-#include <tools/fastfile.hpp>
-#include <tools/ff/fastfile_handlers.hpp>
-#include <games/bo4/pool.hpp>
+#include <tools/ff/linkers/linker_bo4.hpp>
 
-namespace {
-	using namespace fastfile;
+namespace fastfile::linker::bo4 {
+	std::vector<LinkerWorker*>& GetWorkers() {
+		static std::vector<LinkerWorker*> workers{};
+		return workers;
+	}
 
-	struct Asset {
-		games::bo4::pool::XAssetType type;
-		void* location;
-	};
-
-	struct BO4LinkContext {
-		FastFileLinkerContext& linkCtx;
-		std::vector<byte> assetData{};
-		std::vector<const char*> strings{};
-		std::vector<Asset> assets{};
+	uint64_t HashPathName(const std::filesystem::path& path) {
+		std::filesystem::path p{ path };
+		if (p.has_extension()) {
+			p.replace_extension();
+		}
+		std::string fn{ p.filename().string() };
+		uint64_t r;
+		if (!hash::TryHashPattern(fn.data(), r)) {
+			fn = path.string();
+			r = hash::Hash64(fn.data());
+		}
+		LOG_TRACE("Hash path {} -> 0x{:x}", path.string(), r);
+		return r;
 
-	};
+	}
 
 	class FFLinkerBO4 : public FFLinker {
 	public:
@@ -32,22 +35,61 @@ namespace {
 
 			// load common configs
 			core::config::Config cfg{ ctx.input / "config.json" };
+			cfg.SyncConfig(false);
 
 			std::string ffName{ cfg.GetString("data.name") };
 
 			if (!ffName.empty()) {
 				ctx.ffname = ctx.strs.CloneStr(ffName);
 			}
 
-			BO4LinkContext bo4ctx{ ctx };
+			BO4LinkContext bo4ctx{ ctx, cfg };
 
 
-			// todo: load files into bo4ctx.assetData
+			// load files into bo4ctx.assetData
 
-
-			// todo: add to ctx.data the assets/strings headers
+			for (LinkerWorker* w : GetWorkers()) {
+				LOG_DEBUG("Compute '{}'...", w->id);
+				w->Compute(bo4ctx);
+			}
 
-			throw std::runtime_error("not implemented");
+			if (bo4ctx.error) {
+				throw std::runtime_error("Error when linking fast file data");
+			}
+
+			// add to ctx.data the assets/strings headers
+			XAssetList& assetlist{ utils::Allocate<XAssetList>(ctx.data) };
+
+			// write header
+			if (bo4ctx.strings.size()) {
+				assetlist.stringList.count = (int)bo4ctx.strings.size();
+				assetlist.stringList.strings = fastfile::ALLOC_PTR;
+			}
+
+			if (bo4ctx.assets.size()) {
+				assetlist.assetCount = (int)bo4ctx.assets.size();
+				assetlist.assets = fastfile::ALLOC_PTR;
+			}
+
+			if (bo4ctx.strings.size()) {
+				// write string ref array
+				uintptr_t* ptrs{ utils::AllocateArray<uintptr_t>(ctx.data, bo4ctx.strings.size()) };
+				for (size_t i = 0; i < bo4ctx.strings.size(); i++) {
+					ptrs[i] = fastfile::ALLOC_PTR;
+				}
+
+				// write strings
+				for (const char* str : bo4ctx.strings) {
+					utils::WriteString(ctx.data, str);
+				}
+			}
+			if (bo4ctx.assets.size()) {
+				// write asset array
+				utils::WriteValue(ctx.data, bo4ctx.assets.data(), bo4ctx.assets.size() * sizeof(bo4ctx.assets[0]));
+			}
+
+			// write asset data
+			utils::WriteValue(ctx.data, bo4ctx.assetData.data(), bo4ctx.assetData.size());
 		}
 
 	};

src/acts/tools/ff/linkers/linker_bo4.hpp

@@ -0,0 +1,47 @@
+#pragma once
+#include <core/config.hpp>
+#include <tools/ff/fastfile_handlers.hpp>
+#include <games/bo4/pool.hpp>
+
+namespace fastfile::linker::bo4 {
+	struct XHash {
+		uint64_t hash;
+		uint64_t nulled;
+	};
+	struct XAsset {
+		games::bo4::pool::XAssetType type;
+		uintptr_t header; // XAssetHeader
+	};
+
+	struct ScriptStringList {
+		int count;
+		uintptr_t strings; // const char**
+	};
+
+	struct XAssetList {
+		ScriptStringList stringList;
+		int assetCount;
+		uintptr_t assets; // XAsset*
+	};
+
+	struct BO4LinkContext {
+		FastFileLinkerContext& linkCtx;
+		core::config::Config& ffConfig;
+		std::vector<byte> assetData{};
+		std::vector<const char*> strings{};
+		std::vector<XAsset> assets{};
+		bool error{};
+	};
+
+	class LinkerWorker {
+	public:
+		const char* id;
+		LinkerWorker(const char* id) : id(id) {}
+
+		virtual void Compute(BO4LinkContext& ctx) = 0;
+	};
+
+	uint64_t HashPathName(const std::filesystem::path& path);
+
+	std::vector<LinkerWorker*>& GetWorkers();
+}

src/shared/utils/utils.hpp

@@ -273,6 +273,16 @@ namespace utils {
 	T& Allocate(std::vector<byte>& data) {
 		return *reinterpret_cast<T*>(&data[Allocate(data, sizeof(T))]);
 	}
+	/*
+	 * Allocate an array pointer inside a vector
+	 * @param data buffer
+	 * @param count count
+	 * @return pointer
+	 */
+	template<typename T>
+	T* AllocateArray(std::vector<byte>& data, size_t count) {
+		return reinterpret_cast<T*>(&data[Allocate(data, sizeof(T) * count)]);
+	}
 	/*
 	 * Write a padding into a stream
 	 * @param out stream

test/ff-linker/core_test/acts/test/rawfile.txt

@@ -0,0 +1 @@
+this is a raw file with some data!

test/ff-linker/core_test/config.json

@@ -0,0 +1,5 @@
+{
+  "data": {
+    "name": "core_test"
+  }
+}

test/ff-linker/core_test/scripts/core_common/acts/test_shared.csc

@@ -0,0 +1,8 @@
+#using scripts\core_common\array_shared;
+#using scripts\core_common\system_shared;
+
+#namespace test_shared;
+
+function test_function() {
+    return "csc";
+}