From f981736038a00bfcc89242a57a824edc15438bb6 Mon Sep 17 00:00:00 2001 From: Lukas Cone Date: Wed, 31 Jul 2024 19:10:21 +0200 Subject: [PATCH] wip --- 3rd_party/fragmented | 2 +- 3rd_party/spike | 2 +- CMakeLists.txt | 2 + common/include/insomnia/classes/moby.hpp | 49 +- common/include/insomnia/classes/resource.hpp | 163 ++ common/include/insomnia/classes/shader.hpp | 93 +- common/include/insomnia/classes/tie.hpp | 74 +- common/include/insomnia/insomnia.hpp | 8 +- common/include/insomnia/internal/vertex.hpp | 108 ++ common/src/reflected.cpp | 6 +- common/src/serialize.cpp | 357 ++++- effect/extract_effect.cpp | 27 +- extract/extract.cpp | 41 +- extract/gltf.cpp | 61 +- levelmain/CMakeLists.txt | 22 + levelmain/extract.cpp | 1444 ++++++++++++++++++ main/CMakeLists.txt | 22 + main/extract.cpp | 1382 +++++++++++++++++ 18 files changed, 3732 insertions(+), 131 deletions(-) create mode 100644 common/include/insomnia/internal/vertex.hpp create mode 100644 levelmain/CMakeLists.txt create mode 100644 levelmain/extract.cpp create mode 100644 main/CMakeLists.txt create mode 100644 main/extract.cpp diff --git a/3rd_party/fragmented b/3rd_party/fragmented index 51bbc39..82f41f8 160000 --- a/3rd_party/fragmented +++ b/3rd_party/fragmented @@ -1 +1 @@ -Subproject commit 51bbc39b01af194535dce5852b8a4391524c2be2 +Subproject commit 82f41f8565c0f833ca93f885c5ac98041758c353 diff --git a/3rd_party/spike b/3rd_party/spike index 8d9d2c2..fea2872 160000 --- a/3rd_party/spike +++ b/3rd_party/spike @@ -1 +1 @@ -Subproject commit 8d9d2c24f30a7b04a85bc8a61d142877bf7df82e +Subproject commit fea287259c40ce57b386fdf69669497483044087 diff --git a/CMakeLists.txt b/CMakeLists.txt index b236772..9978071 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,6 +21,8 @@ add_subdirectory(common) target_link_libraries(spike_cli insomnia-objects) add_spike_subdir(extract) add_spike_subdir(effect) +add_spike_subdir(levelmain) +add_spike_subdir(main) set(TPD_PATH ${CMAKE_CURRENT_SOURCE_DIR}/3rd_party/fragmented/3rd_party) add_spike_subdir(3rd_party/fragmented/psarc) diff --git a/common/include/insomnia/classes/moby.hpp b/common/include/insomnia/classes/moby.hpp index 870d976..8b2f1b0 100644 --- a/common/include/insomnia/classes/moby.hpp +++ b/common/include/insomnia/classes/moby.hpp @@ -19,7 +19,7 @@ #include "insomnia/internal/base.hpp" #include "spike/type/matrix44.hpp" -struct Primitive : CoreClass { +struct PrimitiveV2 : CoreClass { static constexpr uint32 ID = 0xdd00; uint32 indexOffset; @@ -42,7 +42,7 @@ struct Primitive : CoreClass { struct Mesh : CoreClass { static constexpr uint32 ID = 0xd700; - es::PointerX86 primitives; + es::PointerX86 primitives; uint32 numPrimitives; }; @@ -67,7 +67,7 @@ struct Skeleton : CoreClass { uint32 off2; }; -struct Moby : CoreClass { +struct MobyV2 : CoreClass { static constexpr uint32 ID = 0xd100; float unk00[4]; @@ -102,3 +102,46 @@ struct Moby : CoreClass { es::PointerX86 unkData6; uint32 null01[12]; }; + +struct PrimitiveV1 : CoreClass { + static constexpr uint32 ID = 0xdd00; + + uint16 materialIndex; + uint16 numVertices; + uint16 numIndices; + uint8 numJoints; + uint8 vertexFormat; + uint32 indexOffset; + uint32 vertexBufferOffset; + es::PointerX86 joints; + uint32 unk[3]; +}; + +struct MeshV1 : CoreClass { + es::PointerX86 primitives; + uint32 numPrimitives; +}; + +struct MobyV1 : CoreClass { + static constexpr uint32 ID = 0xd100; + + float unk00[4]; + uint16 unk01; + uint16 unk02; + uint16 numBones; + uint16 unk03; + uint16 numMeshes; + uint16 mobyId; + uint16 null00; + uint8 anotherSet; // bool? + uint8 null01; + es::PointerX86 skeleton; + es::PointerX86 unkData0; + es::PointerX86 meshes; + es::PointerX86 unkData1; + uint32 null02; + uint32 indexBufferOffset; + int32 vertexBufferOffset; + float meshScale; + uint32 unkRest[32]; +}; diff --git a/common/include/insomnia/classes/resource.hpp b/common/include/insomnia/classes/resource.hpp index fd36e9d..b004087 100644 --- a/common/include/insomnia/classes/resource.hpp +++ b/common/include/insomnia/classes/resource.hpp @@ -47,6 +47,20 @@ static constexpr uint32 ResourceTiePathLookupId = 0x3410; static constexpr uint32 ResourceShrubPathLookupId = 0xb700; static constexpr uint32 ResourceCinematicPathLookupId = 0x17d00; +template struct ResourceNameLookup : CoreClass { + static constexpr uint32 ID = id_; + Hash hash; + es::PointerX86 path; + uint32 unk; + + bool operator<(const Hash other) const { return hash < other; } + bool operator==(const Hash other) const { return hash == other; } + bool operator==(uint32 other) const { return hash.part2 == other; } +}; + +using ResourceMobyPath = ResourceNameLookup<0x9480>; +using ResourceTiePath = ResourceNameLookup<0x9280>; + struct EffectTextureBuffer : CoreClass { static constexpr uint32 ID = 0x5300; char data; @@ -61,3 +75,152 @@ struct IndexBuffer : CoreClass { static constexpr uint32 ID = 0xe100; uint16 data; }; + +struct LevelVertexBuffer : CoreClass { + static constexpr uint32 ID = 0x9000; + char data; +}; + +struct LevelIndexBuffer : CoreClass { + static constexpr uint32 ID = 0x9100; + uint16 data; +}; + +struct RegionMesh : CoreClass { + static constexpr uint32 ID = 0x6200; + + float unk[16]; + Vector position; + uint16 materialIndex; + uint16 unk6; + uint32 unk3; + uint32 indexOffset; + uint32 vertexOffset; + uint16 numIndices; + uint16 numVerties; + uint32 unk2[5]; + float meshScale; + uint32 unk4[2]; +}; + +struct RegionMeshV2 : CoreClass { + static constexpr uint32 ID = 0x6200; + + float unk[16]; + uint32 indexOffset; + uint32 vertexOffset; + uint16 numIndices; + uint16 numVerties; + uint16 unk7[2]; + uint16 materialIndex; + uint16 unk6; + uint32 unk2[3]; + Vector position; + uint32 unk4; + float unk5[4]; +}; + +struct ShrubPrimitive : CoreClass { + static constexpr uint32 ID = 0xB200; + uint16 materialIndex; + uint16 unk1; + uint16 numVertices; + uint16 numIndices; + uint32 vertexBufferOffset; + uint32 indexOffset; + uint16 unk4[8]; +}; + +struct Shrub : CoreClass { + static constexpr uint32 ID = 0xB300; + float unk0[20]; + es::PointerX86 primitives; + uint32 numPrimitives; + uint32 null00[2]; + uint16 unk1[2]; + float meshScale; + float unk2[6]; +}; + +struct ShrubInstance : CoreClass { + static constexpr uint32 ID = 0x9500; + es::Matrix44 tm; + float unk0[4]; + es::PointerX86 shrub; + int32 unk1[11]; +}; + +struct FoliageBranchLod { + uint32 indexOffset; + uint16 numIndices; + uint16 unk; +}; + +struct SpriteLodRange { + uint32 indexBegin; + uint32 indexEnd; + uint32 unk0; + float unk1; +}; + +struct SpriteRange { + uint16 indexBegin; + uint16 indexEnd; + uint16 positionsOffset; + uint16 numSprites; +}; + +struct Foliage : CoreClass { + static constexpr uint32 ID = 0xc200; + uint32 unk0; + uint16 unk4; + uint16 unk6; + uint32 textureIndex; + uint32 unk5; + uint32 indexOffset; + uint32 null0; + uint32 branchVertexOffset; + uint32 unk1; + FoliageBranchLod branchLods[4]; + uint32 spriteVertexOffset; + uint32 usedSpriteLods; + SpriteLodRange spriteLodRanges[6]; + float unk2[4]; + es::PointerX86 spritePositions; + uint32 usedSpriteRanges; + SpriteRange spriteRanges[8]; + float unk3[8]; +}; + +struct FoliageInstance : CoreClass { + static constexpr uint32 ID = 0x9700; + + es::Matrix44 tm; + float unk0[33]; + es::PointerX86 foliage; + uint32 unk1[2]; + uint32 unk[4]; +}; + +struct HighmipTextureData : CoreClass { + static constexpr uint32 ID = 0x9800; + + uint32 dataOffset; + uint16 dataSize; + uint16 textureIndex; + int16 unk0; + uint16 unk2; + uint32 null0; +}; + +struct PlantPrimitive : CoreClass { + static constexpr uint32 ID = 0xC700; + + uint32 vertexBufferOffset; + uint32 indexOffset; + uint32 numIndices; + uint32 unk0; + float unk1[4]; + uint16 materialIndex; + uint32 unk2[3]; +}; diff --git a/common/include/insomnia/classes/shader.hpp b/common/include/insomnia/classes/shader.hpp index 1dd998e..a4b5ca1 100644 --- a/common/include/insomnia/classes/shader.hpp +++ b/common/include/insomnia/classes/shader.hpp @@ -29,16 +29,6 @@ struct TextureResource : CoreClass { bool operator==(uint32 other) const { return hash == other; } }; -struct Material : CoreClass { - static constexpr uint32 ID = 0x5000; - int32 diffuseMapId; - int32 normalMapId; - int32 specularMapId; - int32 detailMapId; - uint32 unk0; // flags? - Vector4A16 values[6]; -}; - struct TextureFlags { using location = BitMemberDecl<3, 2>; using isCubemap = BitMemberDecl<2, 1>; @@ -119,12 +109,24 @@ struct TextureControl3 { using TextureControl3Type = TextureControl3::Type; +enum class TextureFormat : uint8 { + R8 = 0x81, + RGB5A1, + RGBA4, + R5G6B5, + RGBA8, + BC1, + BC2, + BC3, + RG8 = 0x8B +}; + // NV4097_SET_*TEXTURE_* registry dump struct Texture : CoreClass { static constexpr uint32 ID = 0x5200; uint32 offset; uint16 numMips; - uint8 format; + TextureFormat format; TextureFlagsType flags; TextureAddressType address; TextureControl0Type control0; @@ -135,7 +137,74 @@ struct Texture : CoreClass { uint32 borderColor; }; -struct ShaderResource : CoreClass { +struct LightmapTexture : Texture { + static constexpr uint32 ID = 0x5400; +}; + +struct ShadowmapTexture : Texture { + static constexpr uint32 ID = 0x5410; +}; + +struct LightmapTextureV1 : Texture { + static constexpr uint32 ID = 0x5500; +}; + +struct TextureV1 : Texture { + static constexpr uint32 ID = 0x5300; +}; + +struct BlendmapTextureV1 : Texture { + static constexpr uint32 ID = 0x5800; +}; + +struct MaterialV1 : CoreClass { + static constexpr uint32 ID = 0x5001; + + float unk0[5]; + uint32 unk1[2]; + bool unkFlag : 1; + bool useSpecular : 1; + bool useGlossiness : 1; + bool useNormalMap : 1; + bool useDetailMap : 1; + bool restFlags : 3; + uint8 blendMode; + uint8 unk2; + uint8 null0; + es::PointerX86 textures[4]; + Vector4A16 values[5]; +}; + +struct MaterialV1_5 : CoreClass { + static constexpr uint32 ID = 0x5000; + + es::PointerX86 diffuse; + es::PointerX86 normal; + es::PointerX86 specular; + es::PointerX86 detail; + bool unkFlag : 1; + bool useSpecular : 1; + bool useGlossiness : 1; + bool useNormalMap : 1; + bool useDetailMap : 1; + bool restFlags : 3; + uint8 blendMode; + uint8 unk0; + uint8 unk1; + Vector4A16 values[6]; +}; + +struct Material : CoreClass { + static constexpr uint32 ID = 0x5000; + int32 diffuseMapId; + int32 normalMapId; + int32 specularMapId; + int32 detailMapId; + uint32 unk0; // flags? + Vector4A16 values[6]; +}; + +struct MaterialResourceNameLookup : CoreClass { static constexpr uint32 ID = 0x5d00; Hash hash; es::PointerX86 lookupPath; diff --git a/common/include/insomnia/classes/tie.hpp b/common/include/insomnia/classes/tie.hpp index 6671361..7ca6972 100644 --- a/common/include/insomnia/classes/tie.hpp +++ b/common/include/insomnia/classes/tie.hpp @@ -18,7 +18,7 @@ #pragma once #include "insomnia/internal/base.hpp" -struct TiePrimitive : CoreClass { +struct TiePrimitiveV2 : CoreClass { static constexpr uint32 ID = 0x3300; uint32 indexOffset; @@ -41,10 +41,10 @@ struct TiePrimitive : CoreClass { uint32 unk3; }; -struct Tie : CoreClass { +struct TieV2 : CoreClass { static constexpr uint32 ID = 0x3400; - es::PointerX86 primitives; + es::PointerX86 primitives; es::PointerX86 unkData1; uint32 unk00; uint32 numPrimitives; @@ -67,3 +67,71 @@ struct TieIndexBuffer : CoreClass { static constexpr uint32 ID = 0x3200; uint16 data; }; + +struct TiePrimitiveV1 : CoreClass { + static constexpr uint32 ID = 0x3300; + + uint16 materialIndex; + uint16 unk1; + uint32 indexOffset; + uint16 numIndices; + uint16 numVertices; + uint16 vertexOffset0; + uint16 vertexOffset1; + uint32 unk[4]; +}; + +struct TieV1 : CoreClass { + static constexpr uint32 ID = 0x3400; + + es::PointerX86 primitives; + es::PointerX86 unkData1; + uint16 numMeshes; + uint16 unk01; + uint32 unk02; + uint32 unk13; + uint32 unk14; + uint32 offset0; + uint32 null00; + Vector meshScale; + float unk03[5]; +}; + +struct TieInstanceV1 : CoreClass { + static constexpr uint32 ID = 0x9300; + + es::Matrix44 tm; + float unk0[16]; + uint32 unk1[3]; + es::PointerX86 tie; + uint32 unk[12]; +}; + +struct TieV1_5 : CoreClass { + static constexpr uint32 ID = 0x3400; + + es::PointerX86 primitives; + es::PointerX86 unkData1; + uint32 unk00; + uint32 numMeshes; + uint32 unk01; + uint32 vertexBufferOffset0; + uint32 vertexBufferOffset1; + uint16 unk13; + uint16 unk14; + Vector meshScale; + float unk03[13]; + uint32 unk04[8]; +}; + +struct TieInstanceV2 : CoreClass { + static constexpr uint32 ID = 0x9240; + + es::Matrix44 tm; + float unk0[4]; // probably bounding sphere + es::PointerX86 tie; + es::PointerX86 unk; + uint32 unk5[2]; + float unk2[4]; + int32 unk3[4]; +}; diff --git a/common/include/insomnia/insomnia.hpp b/common/include/insomnia/insomnia.hpp index 7777ebc..e4defd0 100644 --- a/common/include/insomnia/insomnia.hpp +++ b/common/include/insomnia/insomnia.hpp @@ -102,8 +102,14 @@ struct IGHWHeader { uint64 DEADDEAD; }; +enum class Version { + RFOM, + TOD, + V2, +}; + struct IGHW { - void IS_EXTERN FromStream(BinReaderRef_e rd); + void IS_EXTERN FromStream(BinReaderRef_e rd, Version version); auto Header() const { return reinterpret_cast(buffer.data()); } diff --git a/common/include/insomnia/internal/vertex.hpp b/common/include/insomnia/internal/vertex.hpp new file mode 100644 index 0000000..5bab7ac --- /dev/null +++ b/common/include/insomnia/internal/vertex.hpp @@ -0,0 +1,108 @@ +#pragma once +#include "spike/util/endian.hpp" +#include "spike/type/float.hpp" +#include "spike/type/vectors.hpp" + +struct Vertex0 { + int16 position[3]; + int16 boneIndex; + float16 uv[2]; + uint32 normal; + uint32 tangent; +}; + +struct Vertex1 { + int16 position[3]; + int16 unk; + uint8 bones[4]; + uint8 weights[4]; + float16 uv[2]; + uint32 normal; + uint32 tangent; +}; + +struct RegionVertex { + int16 position[4]; + float16 uv0[2]; + float16 uv1[2]; + uint16 normal[2]; + uint16 tangent[2]; + uint16 unk; +}; + +struct RegionVertexV2 { + int16 position[4]; + float16 uv0[2]; + float16 uv1[2]; + uint32 normal; + uint32 tangent; +}; + +struct BranchVertex { + float16 position[4]; + float16 uv[2]; + uint8 tangent[4]; // ?? + uint8 normal[3]; +}; + +struct SpriteVertex { + float16 spriteSize[2]; + USVector2 uv; + uint32 unk; // prolly normal +}; + +struct PlantVertex { + int16 position[4]; + UCVector4 color; + float16 uv[2]; +}; + +inline void FByteswapper(PlantVertex &item) { + FByteswapper(item.position); + FByteswapper(item.uv); +} + +inline void FByteswapper(SpriteVertex &item) { + FByteswapper(item.spriteSize); + FByteswapper(item.unk); + FByteswapper(item.uv); +} + +inline void FByteswapper(RegionVertex &item) { + FByteswapper(item.position); + FByteswapper(item.uv0); + FByteswapper(item.uv1); + FByteswapper(reinterpret_cast(item.normal[0])); + FByteswapper(reinterpret_cast(item.tangent[0])); + FByteswapper(item.unk); +} + +inline void FByteswapper(RegionVertexV2 &item) { + FByteswapper(item.position); + FByteswapper(item.uv0); + FByteswapper(item.uv1); + FByteswapper(item.normal); + FByteswapper(item.tangent); +} + +inline void FByteswapper(Vertex0 &item) { + FByteswapper(item.position); + FByteswapper(item.boneIndex); + FByteswapper(item.uv); + FByteswapper(item.normal); + FByteswapper(item.tangent); +} + +inline void FByteswapper(Vertex1 &item) { + FByteswapper(item.position); + FByteswapper(item.unk); + FByteswapper(item.uv); + FByteswapper(item.normal); + FByteswapper(item.tangent); +} + +inline void FByteswapper(BranchVertex &item) { + FByteswapper(item.position); + FByteswapper(item.uv); +} + diff --git a/common/src/reflected.cpp b/common/src/reflected.cpp index 7d6f339..d081868 100644 --- a/common/src/reflected.cpp +++ b/common/src/reflected.cpp @@ -114,7 +114,7 @@ REFLECT(CLASS(TextureControl3Type), namespace aggregators { void Material(IGHW &main, pugi::xml_node node) { IGHWTOCIteratorConst textures; - IGHWTOCIteratorConst lookups; + IGHWTOCIteratorConst lookups; IGHWTOCIteratorConst textureResources; IGHWTOCIteratorConst<::Material> materials; @@ -123,8 +123,8 @@ void Material(IGHW &main, pugi::xml_node node) { case Texture::ID: textures = i.Iter(); break; - case ShaderResource::ID: - lookups = i.Iter(); + case MaterialResourceNameLookup::ID: + lookups = i.Iter(); break; case TextureResource::ID: textureResources = i.Iter(); diff --git a/common/src/serialize.cpp b/common/src/serialize.cpp index 45ac9d2..5720dfb 100644 --- a/common/src/serialize.cpp +++ b/common/src/serialize.cpp @@ -18,7 +18,17 @@ #include "insomnia/insomnia.hpp" #include "spike/except.hpp" #include "spike/io/binreader_stream.hpp" -#include +#include + +template +void fixupper(CoreClass *data, bool way, std::set &swapped) { + if (swapped.contains(data)) { + return; + } + + swapped.emplace(data); + FByteswapper(*static_cast(data), way); +} template <> void FByteswapper(Hash &input, bool) { FByteswapper(input.part1); @@ -58,7 +68,10 @@ template <> void FByteswapper(Material &input, bool) { FByteswapper(input.diffuseMapId); FByteswapper(input.normalMapId); FByteswapper(input.specularMapId); - FByteswapper(input.unk0); + FByteswapper(input.values); +} + +template <> void FByteswapper(MaterialV1_5 &input, bool) { FByteswapper(input.values); } @@ -75,7 +88,27 @@ template <> void FByteswapper(Texture &input, bool way) { FByteswapper(input.width); } -template <> void FByteswapper(ShaderResource &input, bool) { +template <> void FByteswapper(LightmapTextureV1 &input, bool way) { + FByteswapper(input, way); +} + +template <> void FByteswapper(TextureV1 &input, bool way) { + FByteswapper(input, way); +} + +template <> void FByteswapper(BlendmapTextureV1 &input, bool way) { + FByteswapper(input, way); +} + +template <> void FByteswapper(LightmapTexture &input, bool way) { + FByteswapper(input, way); +} + +template <> void FByteswapper(ShadowmapTexture &input, bool way) { + FByteswapper(input, way); +} + +template <> void FByteswapper(MaterialResourceNameLookup &input, bool) { FByteswapper(input.mapHashes); FByteswapper(input.hash); } @@ -119,10 +152,6 @@ template <> void FByteswapper(ZoneMap &input, bool way) { FByteswapper(static_cast(input), way); } -template void fixupper(CoreClass *data, bool way) { - FByteswapper(*static_cast(data), way); -} - template <> void FByteswapper(Bone &input, bool) { FByteswapper(input.unk); FByteswapper(input.parentIndex); @@ -149,7 +178,7 @@ template <> void FByteswapper(Skeleton &input, bool) { } } -template <> void FByteswapper(Primitive &input, bool) { +template <> void FByteswapper(PrimitiveV2 &input, bool) { FByteswapper(input.indexOffset); FByteswapper(input.vertexOffset); FByteswapper(input.materialIndex); @@ -162,11 +191,20 @@ template <> void FByteswapper(Primitive &input, bool) { FByteswapper(input.unk3); } +template <> void FByteswapper(PrimitiveV1 &input, bool) { + FByteswapper(input.materialIndex); + FByteswapper(input.numVertices); + FByteswapper(input.numIndices); + FByteswapper(input.indexOffset); + FByteswapper(input.vertexBufferOffset); + FByteswapper(input.unk); +} + template <> void FByteswapper(Mesh &input, bool) { FByteswapper(input.numPrimitives); } -template <> void FByteswapper(Moby &input, bool) { +template <> void FByteswapper(MobyV2 &input, bool) { FByteswapper(input.unk00); FByteswapper(input.unk01); FByteswapper(input.numMeshes); @@ -197,7 +235,46 @@ template <> void FByteswapper(Moby &input, bool) { } } -template <> void FByteswapper(TiePrimitive &input, bool) { +template <> void FByteswapper(MeshV1 &input, bool) { + FByteswapper(input.numPrimitives); +} + +template <> void FByteswapper(MobyV1 &input, bool) { + FByteswapper(input.unk00); + FByteswapper(input.unk01); + FByteswapper(input.unk02); + FByteswapper(input.numBones); + FByteswapper(input.unk03); + FByteswapper(input.numMeshes); + FByteswapper(input.mobyId); + FByteswapper(input.null00); + FByteswapper(input.null02); + FByteswapper(input.indexBufferOffset); + FByteswapper(input.vertexBufferOffset); + FByteswapper(input.meshScale); + + const uint32 numMeshes = input.numMeshes * (input.anotherSet + 1); + + for (uint32 i = 0; i < numMeshes; i++) { + FByteswapper(input.meshes[i]); + } +} + +template <> +void fixupper(CoreClass *data, bool way, std::set &swapped) { + if (swapped.contains(data)) { + return; + } + + swapped.emplace(data); + + MobyV1 &input = *static_cast(data); + + FByteswapper(input, false); + fixupper(input.skeleton, way, swapped); +} + +template <> void FByteswapper(TiePrimitiveV2 &input, bool) { FByteswapper(input.indexOffset); FByteswapper(input.vertexOffset0); FByteswapper(input.vertexOffset1); @@ -214,7 +291,7 @@ template <> void FByteswapper(TiePrimitive &input, bool) { FByteswapper(input.unk3); } -template <> void FByteswapper(Tie &input, bool) { +template <> void FByteswapper(TieV2 &input, bool) { FByteswapper(input.unk00); FByteswapper(input.numPrimitives); FByteswapper(input.unk01); @@ -226,10 +303,212 @@ template <> void FByteswapper(Tie &input, bool) { FByteswapper(input.unk04); } +template <> void FByteswapper(TiePrimitiveV1 &input, bool) { + FByteswapper(input.materialIndex); + FByteswapper(input.unk1); + FByteswapper(input.indexOffset); + FByteswapper(input.numIndices); + FByteswapper(input.numVertices); + FByteswapper(input.vertexOffset0); + FByteswapper(input.vertexOffset1); + FByteswapper(input.unk); +} + +template <> void FByteswapper(TieV1 &input, bool) { + FByteswapper(input.numMeshes); + FByteswapper(input.unk01); + FByteswapper(input.unk02); + FByteswapper(input.unk13); + FByteswapper(input.unk14); + FByteswapper(input.offset0); + FByteswapper(input.null00); + FByteswapper(input.meshScale); + FByteswapper(input.unk03); +} + +template <> void FByteswapper(TieInstanceV1 &input, bool) { + FByteswapper(input.tm); + FByteswapper(input.unk0); + FByteswapper(input.unk1); + FByteswapper(input.unk); +} + +template <> void FByteswapper(RegionMesh &input, bool) { + FByteswapper(input.unk); + FByteswapper(input.position); + FByteswapper(input.materialIndex); + FByteswapper(input.unk6); + FByteswapper(input.unk3); + FByteswapper(input.indexOffset); + FByteswapper(input.vertexOffset); + FByteswapper(input.numIndices); + FByteswapper(input.numVerties); + FByteswapper(input.unk2); + FByteswapper(input.meshScale); + FByteswapper(input.unk4); +} + +template <> void FByteswapper(MaterialV1 &input, bool) { + FByteswapper(input.unk0); + FByteswapper(input.unk1); + FByteswapper(input.values); +} + +template <> void FByteswapper(ShrubPrimitive &input, bool) { + FByteswapper(input.materialIndex); + FByteswapper(input.unk1); + FByteswapper(input.numVertices); + FByteswapper(input.numIndices); + FByteswapper(input.vertexBufferOffset); + FByteswapper(input.indexOffset); + FByteswapper(input.unk4); +} + +template <> void FByteswapper(Shrub &input, bool) { + FByteswapper(input.unk0); + FByteswapper(input.numPrimitives); + FByteswapper(input.null00); + FByteswapper(input.unk1); + FByteswapper(input.meshScale); + FByteswapper(input.unk2); +} + +template <> void FByteswapper(ShrubInstance &input, bool) { + FByteswapper(input.tm); + FByteswapper(input.unk0); + FByteswapper(input.unk1); +} + +template <> void FByteswapper(FoliageBranchLod &input, bool) { + FByteswapper(input.indexOffset); + FByteswapper(input.numIndices); + FByteswapper(input.unk); +} + +template <> void FByteswapper(SpriteRange &input, bool) { + FByteswapper(input.indexBegin); + FByteswapper(input.indexEnd); + FByteswapper(input.positionsOffset); + FByteswapper(input.numSprites); +} + +template <> void FByteswapper(SpriteLodRange &input, bool) { + FByteswapper(input.indexBegin); + FByteswapper(input.indexEnd); + FByteswapper(input.unk0); + FByteswapper(input.unk1); +} + +template <> void FByteswapper(Foliage &input, bool) { + FByteswapper(input.textureIndex); + FByteswapper(input.indexOffset); + FByteswapper(input.null0); + FByteswapper(input.branchVertexOffset); + FByteswapper(input.unk1); + FByteswapper(input.branchLods); + FByteswapper(input.spriteVertexOffset); + FByteswapper(input.usedSpriteLods); + FByteswapper(input.spriteLodRanges); + FByteswapper(input.unk2); + FByteswapper(input.usedSpriteRanges); + FByteswapper(input.spriteRanges); + FByteswapper(input.unk3); +} + +struct FoliageSpritePositions : CoreClass { + static constexpr uint32 ID = 0x9150; + Vector4A16 data; +}; + +struct NavmeshPositions : CoreClass { + static constexpr uint32 ID = 0x14100; + Vector4A16 data; +}; + +struct NavmeshPositions2 : CoreClass { + static constexpr uint32 ID = 0x14900; + Vector4A16 data; +}; + +template <> void FByteswapper(FoliageSpritePositions &input, bool) { + FByteswapper(input.data); +} + +template <> void FByteswapper(NavmeshPositions &input, bool) { + FByteswapper(input.data); +} + +template <> void FByteswapper(NavmeshPositions2 &input, bool) { + FByteswapper(input.data); +} + +template <> void FByteswapper(FoliageInstance &input, bool) { + FByteswapper(input.tm); + FByteswapper(input.unk0); + FByteswapper(input.unk1); + FByteswapper(input.unk); +} + +template <> void FByteswapper(HighmipTextureData &input, bool) { + FByteswapper(input.dataOffset); + FByteswapper(input.dataSize); + FByteswapper(input.textureIndex); + FByteswapper(input.unk0); + FByteswapper(input.unk2); + FByteswapper(input.null0); +} + +template <> void FByteswapper(TieV1_5 &input, bool) { + FByteswapper(input.unk00); + FByteswapper(input.numMeshes); + FByteswapper(input.unk01); + FByteswapper(input.vertexBufferOffset0); + FByteswapper(input.vertexBufferOffset1); + FByteswapper(input.unk13); + FByteswapper(input.unk14); + FByteswapper(input.meshScale); + FByteswapper(input.unk03); + FByteswapper(input.unk04); +} + +template <> void FByteswapper(TieInstanceV2 &input, bool) { + FByteswapper(input.tm); + FByteswapper(input.unk0); + FByteswapper(input.unk5); + FByteswapper(input.unk2); + FByteswapper(input.unk3); +} + +template <> void FByteswapper(RegionMeshV2 &input, bool) { + FByteswapper(input.unk); + FByteswapper(input.indexOffset); + FByteswapper(input.vertexOffset); + FByteswapper(input.numIndices); + FByteswapper(input.numVerties); + FByteswapper(input.materialIndex); + FByteswapper(input.unk6); + FByteswapper(input.unk2); + FByteswapper(input.position); + FByteswapper(input.unk4); + FByteswapper(input.unk5); + FByteswapper(input.unk7); +} + +template <> void FByteswapper(PlantPrimitive &item, bool) { + FByteswapper(item.vertexBufferOffset); + FByteswapper(item.indexOffset); + FByteswapper(item.numIndices); + FByteswapper(item.unk0); + FByteswapper(item.unk1); + FByteswapper(item.materialIndex); + FByteswapper(item.unk2); +} + struct ClassInfo { - void (*swap)(CoreClass *, bool); - uint32 size; + uint32 id; + uint16 size; bool openEnded; + void (*swap)(CoreClass *, bool, std::set &); }; template @@ -238,22 +517,31 @@ template constexpr static bool is_open_ended_v = es::is_detected_v; template auto RegisterClasses() { - return std::map{ - {C::ID, {fixupper, sizeof(C), is_open_ended_v}}...}; + return std::vector{ + {C::ID, sizeof(C), is_open_ended_v, fixupper}...}; } -static const std::map FIXUPS{ +static const std::vector FIXUPS[]{ + RegisterClasses(), + RegisterClasses(), RegisterClasses< ResourceLighting, ResourceZones, ResourceAnimsets, ResourceMobys, ResourceShrubs, ResourceTies, ResourceFoliages, ResourceCubemap, ResourceShaders, ResourceHighmips, ResourceTextures, ResourceCinematics, - TextureResource, Material, Texture, ShaderResource, + TextureResource, Material, Texture, MaterialResourceNameLookup, ShaderResourceLookup, ZoneHash, ZoneNameLookup, ZoneLightmap, - ZoneShadowMap, ZoneDataLookup, ZoneData2Lookup, ZoneData, ZoneMap, Moby, - Primitive, Tie, TiePrimitive>(), + ZoneShadowMap, ZoneDataLookup, ZoneData2Lookup, ZoneData, ZoneMap, + MobyV2, PrimitiveV2, TieV2, TiePrimitiveV2>(), }; -void IGHW::FromStream(BinReaderRef_e rd) { +void IGHW::FromStream(BinReaderRef_e rd, Version version) { rd.SwapEndian(true); IGHWHeader hdr; rd.Push(); @@ -275,6 +563,7 @@ void IGHW::FromStream(BinReaderRef_e rd) { rd.ReadContainer(fixups, hdr.numFixups); for (auto f : fixups) { + f &= 0xfffffff; auto ptr = reinterpret_cast *>(&buffer[0] + f); FByteswapper(*ptr); ptr->Fixup(buffer.data()); @@ -283,12 +572,26 @@ void IGHW::FromStream(BinReaderRef_e rd) { FByteswapper(*Header()); + std::set swapped; + for (auto &item : *this) { FByteswapper(item, false); item.data.Fixup(buffer.data()); - auto found = FIXUPS.find(item.id); + auto &fixups = FIXUPS[int(version)]; + auto found = + std::find_if(fixups.begin(), fixups.end(), [&](const ClassInfo &cls) { + if (cls.id != item.id) { + return false; + } + + if (item.count.ArrayType() == IGHWTOCArrayType::Array) { + return item.size == cls.size; + } + + return true; + }); - if (!es::IsEnd(FIXUPS, found)) { + if (!es::IsEnd(fixups, found)) { char *start = reinterpret_cast(item.data.operator->()); char *end = nullptr; @@ -297,8 +600,8 @@ void IGHW::FromStream(BinReaderRef_e rd) { end = reinterpret_cast(start) + item.size; break; case IGHWTOCArrayType::Array: - end = reinterpret_cast(start) + - item.count.Count() * found->second.size; + end = + reinterpret_cast(start) + item.count.Count() * found->size; break; default: @@ -306,10 +609,10 @@ void IGHW::FromStream(BinReaderRef_e rd) { } while (start < end) { - found->second.swap(reinterpret_cast(start), false); - start += found->second.size; + found->swap(reinterpret_cast(start), false, swapped); + start += found->size; - if (found->second.openEnded) { + if (found->openEnded) { break; } } diff --git a/effect/extract_effect.cpp b/effect/extract_effect.cpp index a7f63a0..3f16291 100644 --- a/effect/extract_effect.cpp +++ b/effect/extract_effect.cpp @@ -20,6 +20,7 @@ #include "spike/app_context.hpp" #include "spike/except.hpp" #include "spike/io/binreader_stream.hpp" +#include static AppInfo_s appInfo{ .header = EffectExtract_DESC " v" EffectExtract_VERSION @@ -33,31 +34,31 @@ void ExtractTexture(AppExtractContext *ctx, std::string path, TexelTile tile = TexelTile::Linear; auto GetFormat = [&] { + using T = TextureFormat; switch (info.format) { - case 0x85: + case T::RGBA8: tile = TexelTile::Morton; return TexelInputFormatType::RGBA8; - case 0x81: + case T::R8: + tile = TexelTile::Morton; return TexelInputFormatType::R8; - case 0x86: - case 0xa6: // srgb??? + case T::BC1: return TexelInputFormatType::BC1; - case 0x88: - case 0xa8: + case T::BC3: return TexelInputFormatType::BC3; - case 0x87: - case 0xa7: + case T::BC2: return TexelInputFormatType::BC2; - case 0x84: + case T::R5G6B5: tile = TexelTile::Morton; return TexelInputFormatType::R5G6B5; - case 0x83: + case T::RGBA4: tile = TexelTile::Morton; return TexelInputFormatType::RGBA4; - case 0x8b: + case T::RG8: tile = TexelTile::Morton; return TexelInputFormatType::RG8; default: + PrintError("Invalid texture format: " + std::to_string(int(info.format))); break; } @@ -84,12 +85,12 @@ void ExtractTexture(AppExtractContext *ctx, std::string path, void AppProcessFile(AppContext *ctx) { BinReaderRef_e rd(ctx->GetStream()); IGHW main; - main.FromStream(rd); + main.FromStream(rd, Version::V2); auto dataStream = ctx->RequestFile(std::string(ctx->workingFile.GetFolder()) + "vfx_system_texel.dat"); BinReaderRef_e rdd(*dataStream.Get()); IGHW data; - data.FromStream(rdd); + data.FromStream(rdd, Version::V2); auto ectx = ctx->ExtractContext(); IGHWTOCIteratorConst texturResources; diff --git a/extract/extract.cpp b/extract/extract.cpp index 6f2a7b8..45e4a7c 100644 --- a/extract/extract.cpp +++ b/extract/extract.cpp @@ -116,9 +116,9 @@ void ExtractShaders(AppContext *ctx, const char *shaderPath = nullptr; BinReaderRef_e rd(*stream.Get()); rd.SetRelativeOrigin(item.offset); - main.FromStream(rd); + main.FromStream(rd, Version::V2); IGHWTOCIteratorConst textures; - IGHWTOCIteratorConst lookups; + IGHWTOCIteratorConst lookups; IGHWTOCIteratorConst textureResources; CatchClasses(main, textures, lookups, textureResources); @@ -225,31 +225,32 @@ void ExtractTexture(AppExtractContext *ctx, std::string path, TexelTile tile = TexelTile::Linear; auto GetFormat = [&] { + using T = TextureFormat; switch (info.format) { - case 0x85: + case T::RGBA8: tile = TexelTile::Morton; return TexelInputFormatType::RGBA8; - case 0x81: + case T::R8: + tile = TexelTile::Morton; return TexelInputFormatType::R8; - case 0x86: - case 0xa6: // srgb??? + case T::BC1: return TexelInputFormatType::BC1; - case 0x88: - case 0xa8: + case T::BC3: return TexelInputFormatType::BC3; - case 0x87: - case 0xa7: + case T::BC2: return TexelInputFormatType::BC2; - case 0x84: + case T::R5G6B5: tile = TexelTile::Morton; return TexelInputFormatType::R5G6B5; - case 0x83: + case T::RGBA4: tile = TexelTile::Morton; return TexelInputFormatType::RGBA4; - case 0x8b: + case T::RG8: tile = TexelTile::Morton; return TexelInputFormatType::RG8; default: + PrintError("Invalid texture format: " + + std::to_string(int(info.format))); break; } @@ -321,7 +322,7 @@ void ExtractZones(AppContext *ctx, AppContextStream regionStream = std::move( ctx->FindFile(std::string(ctx->workingFile.GetFolder()), "^region.dat$")); IGHW region; - region.FromStream(*regionStream.Get()); + region.FromStream(*regionStream.Get(), Version::V2); IGHWTOCIteratorConst zoneHashes; IGHWTOCIteratorConst zoneNames; @@ -375,7 +376,7 @@ void ExtractZones(AppContext *ctx, IGHW curZone; zonesData.SetRelativeOrigin(item.offset); - curZone.FromStream(zonesData); + curZone.FromStream(zonesData, Version::V2); IGHWTOCIteratorConst zoneLightmaps; IGHWTOCIteratorConst zoneShadowmaps; IGHWTOCIteratorConst zoneDataLookups; @@ -443,8 +444,8 @@ void ExtractAnimSets(AppContext *ctx, IGHWTOCIteratorConst mobys, BinReaderRef_e subRd(*stream.Get()); subRd.SetRelativeOrigin(moby.offset); IGHW item; - item.FromStream(subRd); - IGHWTOCIteratorConst model; + item.FromStream(subRd, Version::V2); + IGHWTOCIteratorConst model; CatchClasses(item, model); const Hash animHash = model.at(0).animset; @@ -508,7 +509,7 @@ void ExtractMobys(AppContext *ctx, for (auto &subItem : mobys) { BinReaderRef_e subRd(*stream.Get()); subRd.SetRelativeOrigin(subItem.offset); - main.FromStream(*stream.Get()); + main.FromStream(*stream.Get(), Version::V2); MobyToGltf(shaders, main, ctx, shdStream); } } @@ -526,7 +527,7 @@ void ExtractTies(AppContext *ctx, for (auto &subItem : mobys) { BinReaderRef_e subRd(*stream.Get()); subRd.SetRelativeOrigin(subItem.offset); - main.FromStream(*stream.Get()); + main.FromStream(*stream.Get(), Version::V2); TieToGltf(shaders, main, ctx, shdStream); } } @@ -534,7 +535,7 @@ void ExtractTies(AppContext *ctx, void AppProcessFile(AppContext *ctx) { BinReaderRef_e rd(ctx->GetStream()); IGHW main; - main.FromStream(rd); + main.FromStream(rd, Version::V2); char uniBuffer[0x80000]{}; auto ectx = ctx->ExtractContext(); diff --git a/extract/gltf.cpp b/extract/gltf.cpp index 418a58a..7bf950f 100644 --- a/extract/gltf.cpp +++ b/extract/gltf.cpp @@ -1,45 +1,12 @@ #include "spike/gltf.hpp" #include "insomnia/insomnia.hpp" +#include "insomnia/internal/vertex.hpp" #include "spike/app_context.hpp" #include "spike/io/binreader_stream.hpp" -struct Vertex0 { - int16 position[3]; - int16 boneIndex; - int16 uv[2]; - uint32 normal; - uint32 tangent; -}; - -struct Vertex1 { - int16 position[3]; - int16 unk; - uint8 bones[4]; - uint8 weights[4]; - int16 uv[2]; - uint32 normal; - uint32 tangent; -}; - -void FByteswapper(Vertex0 &item) { - FByteswapper(item.position); - FByteswapper(item.boneIndex); - FByteswapper(item.uv); - FByteswapper(item.normal); - FByteswapper(item.tangent); -} - -void FByteswapper(Vertex1 &item) { - FByteswapper(item.position); - FByteswapper(item.unk); - FByteswapper(item.uv); - FByteswapper(item.normal); - FByteswapper(item.tangent); -} - void MobyToGltf(IGHWTOCIteratorConst &shaders, IGHW &ighw, AppContext *ctx, AppContextStream &shdStream) { - IGHWTOCIteratorConst mobys; + IGHWTOCIteratorConst mobys; IGHWTOCIteratorConst vertexBuffers; IGHWTOCIteratorConst indexBuffers; IGHWTOCIteratorConst shaderLookups; @@ -65,17 +32,17 @@ void MobyToGltf(IGHWTOCIteratorConst &shaders, IGHW &ighw, if (found != shaders.end()) { BinReaderRef_e rd(*shdStream.Get()); rd.SetRelativeOrigin(found->offset); - shaderMain.FromStream(rd); - IGHWTOCIteratorConst lookups; + shaderMain.FromStream(rd, Version::V2); + IGHWTOCIteratorConst lookups; CatchClasses(shaderMain, lookups); - const ShaderResource *lookup = lookups.begin(); + const MaterialResourceNameLookup *lookup = lookups.begin(); gmat.name = lookup->lookupPath.Get(); } else { gmat.name = std::to_string(lookup.hash.part1); } } - const Moby *moby = mobys.begin(); + const MobyV2 *moby = mobys.begin(); const Skeleton *skeleton = moby->skeleton; for (uint32 i = 0; i < skeleton->numBones; i++) { @@ -106,7 +73,7 @@ void MobyToGltf(IGHWTOCIteratorConst &shaders, IGHW &ighw, for (uint32 i = 0; i < moby->numMeshes; i++) { const Mesh &mesh = moby->meshes[i]; for (uint32 p = 0; p < mesh.numPrimitives; p++) { - const Primitive &prim = mesh.primitives[p]; + const PrimitiveV2 &prim = mesh.primitives[p]; if (prim.numJoints < 2) { // continue; @@ -221,7 +188,7 @@ void MobyToGltf(IGHWTOCIteratorConst &shaders, IGHW &ighw, gltf::Mesh &glMesh = main.meshes.emplace_back(); for (uint32 p = 0; p < mesh.numPrimitives; p++) { - const Primitive &prim = mesh.primitives[p]; + const PrimitiveV2 &prim = mesh.primitives[p]; gltf::Primitive &glPrim = glMesh.primitives.emplace_back(); glPrim.material = prim.materialIndex; @@ -325,7 +292,7 @@ void MobyToGltf(IGHWTOCIteratorConst &shaders, IGHW &ighw, void TieToGltf(IGHWTOCIteratorConst &shaders, IGHW &ighw, AppContext *ctx, AppContextStream &shdStream) { - IGHWTOCIteratorConst ties; + IGHWTOCIteratorConst ties; IGHWTOCIteratorConst vertexBuffers; IGHWTOCIteratorConst indexBuffers; IGHWTOCIteratorConst shaderLookups; @@ -351,17 +318,17 @@ void TieToGltf(IGHWTOCIteratorConst &shaders, IGHW &ighw, if (found != shaders.end()) { BinReaderRef_e rd(*shdStream.Get()); rd.SetRelativeOrigin(found->offset); - shaderMain.FromStream(rd); - IGHWTOCIteratorConst lookups; + shaderMain.FromStream(rd, Version::V2); + IGHWTOCIteratorConst lookups; CatchClasses(shaderMain, lookups); - const ShaderResource *lookup = lookups.begin(); + const MaterialResourceNameLookup *lookup = lookups.begin(); gmat.name = lookup->lookupPath.Get(); } else { gmat.name = std::to_string(lookup.hash.part1); } } - const Tie *tie = ties.begin(); + const TieV2 *tie = ties.begin(); const uint16 *indexBuffer = &indexBuffers.begin()->data; const char *vertexBuffer = &vertexBuffers.begin()->data; std::map joints; @@ -389,7 +356,7 @@ void TieToGltf(IGHWTOCIteratorConst &shaders, IGHW &ighw, gltf::Mesh &glMesh = main.meshes.emplace_back(); for (uint32 p = 0; p < tie->numPrimitives; p++) { - const TiePrimitive &prim = tie->primitives[p]; + const TiePrimitiveV2 &prim = tie->primitives[p]; gltf::Primitive &glPrim = glMesh.primitives.emplace_back(); glPrim.material = prim.materialIndex; diff --git a/levelmain/CMakeLists.txt b/levelmain/CMakeLists.txt new file mode 100644 index 0000000..e67970a --- /dev/null +++ b/levelmain/CMakeLists.txt @@ -0,0 +1,22 @@ +cmake_minimum_required(VERSION 3.12) + +project(LevelmainToGLTF) + +build_target( + NAME + levelmain_to_gltf + TYPE + ESMODULE + VERSION + 1 + SOURCES + extract.cpp + LINKS + gltf-interface + insomnia-interface + AUTHOR + "Lukas Cone" + DESCR + "Insomnia Engine levelmain extract" + START_YEAR + 2024) diff --git a/levelmain/extract.cpp b/levelmain/extract.cpp new file mode 100644 index 0000000..8c14a46 --- /dev/null +++ b/levelmain/extract.cpp @@ -0,0 +1,1444 @@ +/* InsomniaToolset LevelmainToGLTF + Copyright(C) 2024 Lukas Cone + + This program is free software : you can redistribute it and / or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program.If not, see . +*/ + +#include "insomnia/insomnia.hpp" +#include "insomnia/internal/vertex.hpp" +#include "nlohmann/json.hpp" +#include "project.h" +#include "spike/app_context.hpp" +#include "spike/except.hpp" +#include "spike/gltf.hpp" +#include "spike/io/binreader_stream.hpp" +#include "spike/master_printer.hpp" +#include "spike/reflect/reflector.hpp" +#include "spike/type/float.hpp" +#include "spike/uni/rts.hpp" +#include + +std::string_view filters[]{ + "^ps3levelmain.dat$", +}; + +static AppInfo_s appInfo{ + .header = LevelmainToGLTF_DESC + " v" LevelmainToGLTF_VERSION ", " LevelmainToGLTF_COPYRIGHT "Lukas Cone", + .filters = filters, +}; + +AppInfo_s *AppInitModule() { return &appInfo; } + +struct IMGLTF : GLTFModel { + GLTFStream &GetTranslations() { + if (instTrs < 0) { + auto &str = NewStream("instance-tms", 20); + instTrs = str.slot; + return str; + } + + return Stream(instTrs); + } + + GLTFStream &GetScales() { + if (instScs < 0) { + auto &str = NewStream("instance-scale"); + instScs = str.slot; + return str; + } + + return Stream(instScs); + } + +private: + int32 instTrs = -1; + int32 instScs = -1; +}; + +struct TextureKey { + const Texture *tex; + int32 id = -1; + bool albedo : 1 = false; + bool normal : 1 = false; + bool gloss : 1 = false; + bool specular : 1 = false; + bool emissive : 1 = false; + bool exponent : 1 = false; + + bool operator<(const TextureKey &o) const { + if (o.emissive == emissive) { + return tex < o.tex; + } + + return emissive < o.emissive; + } +}; + +struct TexStream : TexelOutput { + GLTF &main; + GLTFStream *str = nullptr; + bool ignore = false; + + TexStream(GLTF &main_) : main{main_} {} + + void SendData(std::string_view data) override { + if (str) [[likely]] { + str->wr.WriteContainer(data); + } + } + void NewFile(std::string path) override { + if (ignore) { + return; + } + + str = &main.NewStream(path); + } +}; + +void ExtractTexture(AppContext *ctx, std::string path, + std::istream &textureStream, TextureKey &info, + TexStream *texOut = nullptr) { + thread_local static std::string tmpBuffer(4 * 2048 * 2048, 0); + + textureStream.clear(); + textureStream.seekg(info.tex->offset); + textureStream.read(tmpBuffer.data(), tmpBuffer.size()); + + TexelTile tile = TexelTile::Linear; + + auto GetFormat = [&] { + using T = TextureFormat; + switch (info.tex->format) { + case T::RGBA8: + tile = TexelTile::Morton; + return TexelInputFormatType::RGBA8; + case T::R8: + tile = TexelTile::Morton; + return TexelInputFormatType::R8; + case T::BC1: + return TexelInputFormatType::BC1; + case T::BC3: + return TexelInputFormatType::BC3; + case T::BC2: + return TexelInputFormatType::BC2; + case T::R5G6B5: + tile = TexelTile::Morton; + return TexelInputFormatType::R5G6B5; + case T::RGBA4: + tile = TexelTile::Morton; + return TexelInputFormatType::RGBA4; + case T::RG8: + tile = TexelTile::Morton; + return TexelInputFormatType::RG8; + default: + PrintError("Invalid texture format: " + + std::to_string(int(info.tex->format))); + break; + } + + return TexelInputFormatType::INVALID; + }; + + TexelSwizzle swizzle; + + if (info.normal) { + swizzle.a = TexelSwizzleType::White; + swizzle.r = TexelSwizzleType::Alpha; + swizzle.g = TexelSwizzleType::GreenInverted; + swizzle.b = TexelSwizzleType::DeriveZ; + } else if (info.gloss) { + swizzle.r = swizzle.g = swizzle.b = TexelSwizzleType::RedInverted; + + if (info.specular) { + swizzle.a = TexelSwizzleType::Green; + } else { + swizzle.a = TexelSwizzleType::White; + } + } else if (info.emissive) { + swizzle.r = swizzle.g = swizzle.b = TexelSwizzleType::Blue; + swizzle.a = TexelSwizzleType::White; + } + + NewTexelContextCreate tctx{ + .width = info.tex->width, + .height = info.tex->height, + .baseFormat = + { + .type = GetFormat(), + .swizzle = swizzle, + .tile = tile, + .swapPacked = true, + }, + .depth = std::max( + uint16(1), uint16(info.tex->control3.Get())), + .numMipmaps = uint8(info.tex->numMips), + .data = tmpBuffer.data(), + .texelOutput = texOut, + .formatOverride = + texOut ? TexelContextFormat::UPNG : TexelContextFormat::Config, + }; + + auto EmissiveCheck = [texOut](char *data, uint32 stride, uint32 numTexels) { + texOut->ignore = true; + for (uint32 i = 0; i < numTexels; i++, data += stride) { + if (*data) { + texOut->ignore = false; + return; + } + } + }; + + if (info.emissive) { + tctx.postProcess = EmissiveCheck; + } + + if (texOut) { + ctx->NewImage(tctx); + } else { + ctx->ExtractContext()->NewImage(path + std::to_string(info.id), tctx); + } +} + +int32 TryExtractTexture(AppContext *ctx, GLTF &main, TextureKey key, + const Texture *textures, std::istream &textureStream, + std::set &textureRemaps) { + if (auto found = textureRemaps.find(key); found != textureRemaps.end()) { + return found->id; + } + + gltf::Texture glTexture{}; + glTexture.source = main.textures.size(); + gltf::Image glImage{}; + glImage.mimeType = "image/png"; + glImage.name = "texture_" + std::to_string(std::distance(textures, key.tex)); + + if (key.emissive) { + glImage.name.append("_e"); + } + + TexStream tStr{main}; + + ExtractTexture(ctx, glImage.name, textureStream, key, &tStr); + + if (!tStr.str) { + key.id = -1; + textureRemaps.emplace(key); + return -1; + } + + key.id = glTexture.source; + textureRemaps.emplace(key); + glImage.bufferView = tStr.str->slot; + main.textures.emplace_back(glTexture); + main.images.emplace_back(glImage); + return glTexture.source; +} + +void MakeMaterials(AppContext *ctx, IMGLTF &main, + const std::map &materialRemaps, + IGHWTOCIteratorConst materials, + const Texture *textures, std::istream &textureStream, + std::set &textureRemaps) { + main.materials.resize(materialRemaps.size()); + + for (auto [mid, mindex] : materialRemaps) { + gltf::Material &glMat = main.materials.at(mindex); + glMat.name = "material_" + std::to_string(mid); + glMat.pbrMetallicRoughness.metallicFactor = 0; + const MaterialV1 &mat = materials.at(mid); + const Texture *albedo = mat.textures[0]; + TextureKey albedoInfo{ + .tex = albedo, + .albedo = true, + }; + glMat.pbrMetallicRoughness.baseColorTexture.index = TryExtractTexture( + ctx, main, albedoInfo, textures, textureStream, textureRemaps); + + if (mat.blendMode == 4) { + glMat.alphaMode = gltf::Material::AlphaMode::Mask; + } else if (mat.blendMode) { + glMat.alphaMode = gltf::Material::AlphaMode::Blend; + } + + const Texture *normal = mat.textures[1]; + + if (normal) { + if (!mat.useNormalMap) { + PrintWarning("Material ", mid, " uses normal but no flags"); + } + TextureKey normalInfo{ + .tex = normal, + .normal = true, + }; + + glMat.normalTexture.index = TryExtractTexture( + ctx, main, normalInfo, textures, textureStream, textureRemaps); + } + + const Texture *special = mat.textures[2]; + + if (special) { + if (!mat.useGlossiness && !mat.useSpecular) { + PrintWarning("Material ", mid, " uses special but no flags"); + } else { + TextureKey specInfo{ + .tex = special, + .gloss = mat.useGlossiness, + .specular = mat.useSpecular, + }; + + int32 specId = TryExtractTexture(ctx, main, specInfo, textures, + textureStream, textureRemaps); + + if (mat.useSpecular) { + nlohmann::json &spec = + glMat.GetExtensionsAndExtras()["extensions"] + ["KHR_materials_specular"]; + spec["specularTexture"]["index"] = specId; + } + + if (mat.useGlossiness) { + glMat.pbrMetallicRoughness.metallicRoughnessTexture.index = specId; + } + } + + TextureKey emisKey{ + .tex = special, + .emissive = true, + }; + + glMat.emissiveTexture.index = TryExtractTexture( + ctx, main, emisKey, textures, textureStream, textureRemaps); + } + } +} + +void MakeFoliageMaterials(AppContext *ctx, IMGLTF &main, + const std::map &materialRemaps, + const Texture *textures, std::istream &textureStream, + std::set &textureRemaps) { + main.materials.resize(materialRemaps.size() + main.materials.size()); + + for (auto [mid, mindex] : materialRemaps) { + gltf::Material &glMat = main.materials.at(mindex); + glMat.name = "foliage_material_" + std::to_string(mid); + glMat.doubleSided = true; + const Texture &albedo = textures[mid]; + TextureKey albedoInfo{ + .tex = &albedo, + .albedo = true, + }; + glMat.pbrMetallicRoughness.baseColorTexture.index = TryExtractTexture( + ctx, main, albedoInfo, textures, textureStream, textureRemaps); + glMat.alphaMode = gltf::Material::AlphaMode::Mask; + } +} + +void MobyToGltf(const MobyV1 &moby, AppContext *ctx, BinReaderRef_e stream, + IGHWTOCIteratorConst materials, + const Texture *textures) { + IMGLTF main; + + const Skeleton *skeleton = moby.skeleton; + + for (uint32 i = 0; i < skeleton->numBones; i++) { + gltf::Node &glNode = main.nodes.emplace_back(); + glNode.name = std::to_string(i); + es::Matrix44 tm(skeleton->tms0[i]); + + if (int16 parentIndex = skeleton->bones[i].parentIndex; + parentIndex == int16(i)) { + main.scenes.front().nodes.emplace_back(i); + } else { + main.nodes.at(parentIndex).children.emplace_back(i); + es::Matrix44 ptm(skeleton->tms1[parentIndex]); + tm = ptm * tm; + } + + tm.r1().w = 0; + tm.r2().w = 0; + tm.r3().w = 0; + tm.r4().w = 1; + + memcpy(glNode.matrix.data(), &tm, 64); + } + + std::map joints; + const uint32 numMeshes = moby.numMeshes * (moby.anotherSet + 1); + + for (uint32 i = 0; i < numMeshes; i++) { + const MeshV1 &mesh = moby.meshes[i]; + for (uint32 p = 0; p < mesh.numPrimitives; p++) { + const PrimitiveV1 &prim = mesh.primitives[p]; + + if (prim.numJoints < 2) { + // continue; + } + + for (uint32 j = 0; j < prim.numJoints; j++) { + uint16 joint = prim.joints[j]; + FByteswapper(joint); + joints.try_emplace(joint, joints.size()); + } + } + } + + { + gltf::Skin &skn = main.skins.emplace_back(); + skn.joints.resize(joints.size()); + GLTFStream &ibmStream = main.SkinStream(); + auto [acc, accId] = main.NewAccessor(ibmStream, 16); + acc.type = gltf::Accessor::Type::Mat4; + acc.componentType = gltf::Accessor::ComponentType::Float; + acc.count = joints.size(); + skn.inverseBindMatrices = accId; + std::vector ibms; + ibms.resize(joints.size()); + + for (auto &[jid, idx] : joints) { + skn.joints[idx] = jid; + es::Matrix44 ibm = skeleton->tms1[jid]; + ibm.r1().w = 0; + ibm.r2().w = 0; + ibm.r3().w = 0; + ibm.r4().w = 1; + ibms[idx] = ibm; + } + + ibmStream.wr.WriteContainer(ibms); + } + + struct AttributeBoneIndex : AttributeCodec { + AttributeBoneIndex(const std::map &joints_) + : joints(joints_) {} + void Sample(uni::FormatCodec::fvec &out, const char *input, + size_t stride) const override { + for (auto &o : out) { + int16 index = *reinterpret_cast(input); + uint16 joint = jointMap[std::abs((index + 1) / 3)]; + FByteswapper(joint); + o.x = joints.at(joint); + input += stride; + } + } + void Transform(uni::FormatCodec::fvec &) const override {} + bool CanSample() const override { return true; } + bool CanTransform() const override { return false; } + bool IsNormalized() const override { return false; } + + const std::map &joints; + const uint16 *jointMap = nullptr; + } attributeBoneIndex{joints}; + + struct AttributeBoneIndices : AttributeCodec { + AttributeBoneIndices(const std::map &joints_) + : joints(joints_) {} + void Sample(uni::FormatCodec::fvec &out, const char *input, + size_t stride) const override { + for (auto &o : out) { + UCVector4 index = *reinterpret_cast(input); + for (uint32 i = 0; i < 4; i++) { + uint16 joint = jointMap[index[i]]; + FByteswapper(joint); + o[i] = joints.at(joint); + } + input += stride; + } + } + void Transform(uni::FormatCodec::fvec &) const override {} + bool CanSample() const override { return true; } + bool CanTransform() const override { return false; } + bool IsNormalized() const override { return false; } + + const std::map &joints; + const uint16 *jointMap = nullptr; + } attributeBoneIndices{joints}; + + struct AttributeMul : AttributeCodec { + AttributeMul(float scale) : mul(scale * 0x7fff) {} + void Sample(uni::FormatCodec::fvec &, const char *, size_t) const override { + } + void Transform(uni::FormatCodec::fvec &in) const override { + for (Vector4A16 &v : in) { + v = v * mul; + } + } + bool CanSample() const override { return false; } + bool CanTransform() const override { return true; } + bool IsNormalized() const override { return false; } + + Vector4A16 mul; + } attributeMul{moby.meshScale}; + + std::map materialRemaps; + + for (uint32 i = 0; i < numMeshes; i++) { + const MeshV1 &mesh = moby.meshes[i]; + if (mesh.numPrimitives == 0 || !mesh.primitives) { + continue; + } + + main.scenes.front().nodes.emplace_back(main.nodes.size()); + gltf::Node &glNode = main.nodes.emplace_back(); + glNode.mesh = main.meshes.size(); + glNode.skin = 0; + glNode.name = "Mesh_" + std::to_string(i); + gltf::Mesh &glMesh = main.meshes.emplace_back(); + + for (uint32 p = 0; p < mesh.numPrimitives; p++) { + const PrimitiveV1 &prim = mesh.primitives[p]; + gltf::Primitive &glPrim = glMesh.primitives.emplace_back(); + glPrim.material = + materialRemaps.try_emplace(prim.materialIndex, materialRemaps.size()) + .first->second; + stream.Seek(moby.vertexBufferOffset + prim.vertexBufferOffset); + + if (prim.vertexFormat == 0) { + std::vector vtx0; + stream.ReadContainer(vtx0, prim.numVertices); + + attributeBoneIndex.jointMap = prim.joints; + + Attribute attrs[]{ + { + .type = uni::DataType::R16G16B16, + .format = uni::FormatType::NORM, + .usage = AttributeType::Position, + .customCodec = &attributeMul, + }, + { + .type = uni::DataType::R16, + .format = uni::FormatType::INVALID, + .usage = AttributeType::BoneIndices, + .customCodec = &attributeBoneIndex, + }, + { + .type = uni::DataType::R16G16, + .format = uni::FormatType::FLOAT, + .usage = AttributeType::TextureCoordiante, + }, + { + .type = uni::DataType::R11G11B10, + .format = uni::FormatType::NORM, + .usage = AttributeType::Normal, + }, + }; + + glPrim.attributes = + main.SaveVertices(vtx0.data(), vtx0.size(), attrs, sizeof(Vertex0)); + } else { + std::vector vtx1; + stream.ReadContainer(vtx1, prim.numVertices); + + attributeBoneIndices.jointMap = prim.joints; + + Attribute attrs[]{ + { + .type = uni::DataType::R16G16B16A16, + .format = uni::FormatType::NORM, + .usage = AttributeType::Position, + .customCodec = &attributeMul, + }, + { + .type = uni::DataType::R8G8B8A8, + .format = uni::FormatType::INVALID, + .usage = AttributeType::BoneIndices, + .customCodec = &attributeBoneIndices, + }, + { + .type = uni::DataType::R8G8B8A8, + .format = uni::FormatType::UNORM, + .usage = AttributeType::BoneWeights, + }, + { + .type = uni::DataType::R16G16, + .format = uni::FormatType::FLOAT, + .usage = AttributeType::TextureCoordiante, + }, + { + .type = uni::DataType::R11G11B10, + .format = uni::FormatType::NORM, + .usage = AttributeType::Normal, + }, + }; + + glPrim.attributes = + main.SaveVertices(vtx1.data(), vtx1.size(), attrs, sizeof(Vertex1)); + } + + stream.Seek(moby.indexBufferOffset + prim.indexOffset * 2); + + std::vector idx; + stream.ReadContainer(idx, prim.numIndices); + + glPrim.indices = main.SaveIndices(idx.data(), idx.size()).accessorIndex; + } + } + + std::set textureRemaps; + MakeMaterials(ctx, main, materialRemaps, materials, textures, + stream.BaseStream(), textureRemaps); + + main.FinishAndSave(ctx->NewFile(std::string(ctx->workingFile.GetFolder()) + + "moby_" + std::to_string(moby.mobyId) + + ".glb") + .str, + ""); +} + +void TieToGltf(const TieV1 &tie, IMGLTF &level, + const LevelIndexBuffer &idxBuffer, + const LevelVertexBuffer &vtxBuffer, + IGHWTOCIteratorConst tieInstances, uint32 index, + std::map &materialRemaps) { + const uint16 *indexBuffer = &idxBuffer.data; + const char *vertexBuffer = &vtxBuffer.data; + + level.scenes.front().nodes.emplace_back(level.nodes.size()); + gltf::Node &glNode = level.nodes.emplace_back(); + glNode.mesh = level.meshes.size(); + glNode.name = "TieMesh_" + std::to_string(index); + gltf::Mesh &glMesh = level.meshes.emplace_back(); + + struct AttributeMul : AttributeCodec { + AttributeMul(Vector scale) : mul(scale * 0x7fff) {} + void Sample(uni::FormatCodec::fvec &, const char *, size_t) const override { + } + void Transform(uni::FormatCodec::fvec &in) const override { + for (Vector4A16 &v : in) { + v = v * mul; + } + } + bool CanSample() const override { return false; } + bool CanTransform() const override { return true; } + bool IsNormalized() const override { return false; } + + Vector4A16 mul; + } attributeMul{tie.meshScale}; + + for (uint32 p = 0; p < tie.numMeshes; p++) { + const TiePrimitiveV1 &prim = tie.primitives[p]; + gltf::Primitive &glPrim = glMesh.primitives.emplace_back(); + glPrim.material = + materialRemaps.try_emplace(prim.materialIndex, materialRemaps.size()) + .first->second; + + const uint16 *indices = indexBuffer + prim.indexOffset; + const Vertex0 *vertices = + reinterpret_cast(vertexBuffer + tie.unk13) + + prim.vertexOffset0; + + std::vector vtx0(vertices, vertices + prim.numVertices); + + for (auto &v : vtx0) { + FByteswapper(v); + } + + Attribute attrs[]{ + { + .type = uni::DataType::R16G16B16A16, + .format = uni::FormatType::NORM, + .usage = AttributeType::Position, + .customCodec = &attributeMul, + }, + { + .type = uni::DataType::R16G16, + .format = uni::FormatType::FLOAT, + .usage = AttributeType::TextureCoordiante, + }, + { + .type = uni::DataType::R11G11B10, + .format = uni::FormatType::NORM, + .usage = AttributeType::Normal, + }, + }; + + glPrim.attributes = + level.SaveVertices(vtx0.data(), vtx0.size(), attrs, sizeof(Vertex0)); + + std::vector idx(indices, indices + prim.numIndices); + for (uint16 &i : idx) { + FByteswapper(i); + } + + glPrim.indices = level.SaveIndices(idx.data(), idx.size()).accessorIndex; + } + + std::vector tms; + + for (auto &inst : tieInstances) { + if (inst.tie == &tie) { + tms.emplace_back(inst.tm); + } + } + + if (tms.size() == 1) { + memcpy(glNode.matrix.data(), tms.data(), 64); + } else if (tms.size() > 1) { + std::vector scales; + bool processScales = false; + + auto &str = level.GetTranslations(); + auto [accPos, accPosIndex] = level.NewAccessor(str, 4); + accPos.type = gltf::Accessor::Type::Vec3; + accPos.componentType = gltf::Accessor::ComponentType::Float; + accPos.count = tms.size(); + + auto [accRot, accRotIndex] = level.NewAccessor(str, 4, 12); + accRot.type = gltf::Accessor::Type::Vec4; + accRot.componentType = gltf::Accessor::ComponentType::Short; + accRot.normalized = true; + accRot.count = tms.size(); + Vector4A16::SetEpsilon(0.00001f); + + for (const es::Matrix44 &mtx : tms) { + uni::RTSValue val{}; + mtx.Decompose(val.translation, val.rotation, val.scale); + scales.emplace_back(val.scale); + + if (!processScales) { + processScales = val.scale != Vector4A16(1, 1, 1, 0); + } + + str.wr.Write(val.translation); + + val.rotation.Normalize() *= 0x7fff; + val.rotation = + Vector4A16(_mm_round_ps(val.rotation._data, _MM_ROUND_NEAREST)); + auto comp = val.rotation.Convert(); + str.wr.Write(comp); + } + + auto &attrs = + glNode.GetExtensionsAndExtras()["extensions"]["EXT_mesh_gpu_instancing"] + ["attributes"]; + + attrs["TRANSLATION"] = accPosIndex; + attrs["ROTATION"] = accRotIndex; + + if (processScales) { + auto &str = level.GetScales(); + auto [accScale, accScaleIndex] = level.NewAccessor(str, 4); + accScale.type = gltf::Accessor::Type::Vec3; + accScale.componentType = gltf::Accessor::ComponentType::Float; + accScale.count = tms.size(); + str.wr.WriteContainer(scales); + attrs["SCALE"] = accScaleIndex; + } + } +} + +void ShrubToGltf(const Shrub &shrub, IMGLTF &level, + const LevelIndexBuffer &idxBuffer, + const LevelVertexBuffer &vtxBuffer, + IGHWTOCIteratorConst shrubInstances, + uint32 index, std::map &materialRemaps) { + const uint16 *indexBuffer = &idxBuffer.data; + const char *vertexBuffer = &vtxBuffer.data; + + level.scenes.front().nodes.emplace_back(level.nodes.size()); + gltf::Node &glNode = level.nodes.emplace_back(); + glNode.mesh = level.meshes.size(); + glNode.name = "Shrub_" + std::to_string(index); + gltf::Mesh &glMesh = level.meshes.emplace_back(); + + struct AttributeMul : AttributeCodec { + AttributeMul(Vector scale) : mul(scale) {} + void Sample(uni::FormatCodec::fvec &, const char *, size_t) const override { + } + void Transform(uni::FormatCodec::fvec &in) const override { + for (Vector4A16 &v : in) { + v = v * mul; + } + } + bool CanSample() const override { return false; } + bool CanTransform() const override { return true; } + bool IsNormalized() const override { return false; } + + Vector4A16 mul; + } attributeMul{shrub.meshScale / 0x1000}; + + for (uint32 p = 0; p < shrub.numPrimitives; p++) { + const ShrubPrimitive &prim = shrub.primitives[p]; + gltf::Primitive &glPrim = glMesh.primitives.emplace_back(); + glPrim.material = + materialRemaps.try_emplace(prim.materialIndex, materialRemaps.size()) + .first->second; + + const uint16 *indices = indexBuffer + prim.indexOffset; + const Vertex0 *vertices = reinterpret_cast( + vertexBuffer + prim.vertexBufferOffset); + + std::vector vtx0(vertices, vertices + prim.numVertices); + + for (auto &v : vtx0) { + FByteswapper(v); + } + + Attribute attrs[]{ + { + .type = uni::DataType::R16G16B16A16, + .format = uni::FormatType::NORM, + .usage = AttributeType::Position, + .customCodec = &attributeMul, + }, + { + .type = uni::DataType::R16G16, + .format = uni::FormatType::FLOAT, + .usage = AttributeType::TextureCoordiante, + }, + { + .type = uni::DataType::R11G11B10, + .format = uni::FormatType::NORM, + .usage = AttributeType::Normal, + }, + }; + + glPrim.attributes = + level.SaveVertices(vtx0.data(), vtx0.size(), attrs, sizeof(Vertex0)); + + std::vector idx(indices, indices + prim.numIndices); + for (uint16 &i : idx) { + FByteswapper(i); + } + + glPrim.indices = level.SaveIndices(idx.data(), idx.size()).accessorIndex; + } + + std::vector tms; + + for (auto &inst : shrubInstances) { + if (inst.shrub == &shrub) { + tms.emplace_back(inst.tm); + } + } + + if (tms.size() == 1) { + memcpy(glNode.matrix.data(), tms.data(), 64); + } else if (tms.size() > 1) { + std::vector scales; + bool processScales = false; + + auto &str = level.GetTranslations(); + auto [accPos, accPosIndex] = level.NewAccessor(str, 4); + accPos.type = gltf::Accessor::Type::Vec3; + accPos.componentType = gltf::Accessor::ComponentType::Float; + accPos.count = tms.size(); + + auto [accRot, accRotIndex] = level.NewAccessor(str, 4, 12); + accRot.type = gltf::Accessor::Type::Vec4; + accRot.componentType = gltf::Accessor::ComponentType::Short; + accRot.normalized = true; + accRot.count = tms.size(); + Vector4A16::SetEpsilon(0.00001f); + + for (const es::Matrix44 &mtx : tms) { + uni::RTSValue val{}; + mtx.Decompose(val.translation, val.rotation, val.scale); + scales.emplace_back(val.scale); + + if (!processScales) { + processScales = val.scale != Vector4A16(1, 1, 1, 0); + } + + str.wr.Write(val.translation); + + val.rotation.Normalize() *= 0x7fff; + val.rotation = + Vector4A16(_mm_round_ps(val.rotation._data, _MM_ROUND_NEAREST)); + auto comp = val.rotation.Convert(); + str.wr.Write(comp); + } + + auto &attrs = + glNode.GetExtensionsAndExtras()["extensions"]["EXT_mesh_gpu_instancing"] + ["attributes"]; + + attrs["TRANSLATION"] = accPosIndex; + attrs["ROTATION"] = accRotIndex; + + if (processScales) { + auto &str = level.GetScales(); + auto [accScale, accScaleIndex] = level.NewAccessor(str, 4); + accScale.type = gltf::Accessor::Type::Vec3; + accScale.componentType = gltf::Accessor::ComponentType::Float; + accScale.count = tms.size(); + str.wr.WriteContainer(scales); + attrs["SCALE"] = accScaleIndex; + } + } +} + +void RegionToGltf(IGHWTOCIteratorConst items, IMGLTF &level, + const LevelIndexBuffer &idxBuffer, + const LevelVertexBuffer &vtxBuffer, + std::map &materialRemaps) { + const uint16 *indexBuffer = &idxBuffer.data; + const char *vertexBuffer = &vtxBuffer.data; + + level.scenes.front().nodes.emplace_back(level.nodes.size()); + gltf::Node &glNode = level.nodes.emplace_back(); + glNode.mesh = level.meshes.size(); + glNode.name = "RegionMesh"; + gltf::Mesh &glMesh = level.meshes.emplace_back(); + + for (const RegionMesh &item : items) { + gltf::Primitive &glPrim = glMesh.primitives.emplace_back(); + glPrim.material = + materialRemaps.try_emplace(item.materialIndex, materialRemaps.size()) + .first->second; + + const uint16 *indices = indexBuffer + item.indexOffset; + const RegionVertex *vertices = reinterpret_cast( + vertexBuffer + item.vertexOffset); + + std::vector vtx0(vertices, vertices + item.numVerties); + + for (auto &v : vtx0) { + FByteswapper(v); + } + + AttributeMad attributeMad; + attributeMad.mul = Vector4A16(0x7fff) / 0x100; + attributeMad.add = item.position / 0x100; + + Attribute attrs[]{ + { + .type = uni::DataType::R16G16B16A16, + .format = uni::FormatType::NORM, + .usage = AttributeType::Position, + .customCodec = &attributeMad, + }, + { + .type = uni::DataType::R16G16, + .format = uni::FormatType::FLOAT, + .usage = AttributeType::TextureCoordiante, + }, + { + .type = uni::DataType::R16G16, + .format = uni::FormatType::FLOAT, + .usage = AttributeType::TextureCoordiante, + }, + { + .type = uni::DataType::R11G11B10, + .format = uni::FormatType::NORM, + .usage = AttributeType::Normal, + }, + }; + + glPrim.attributes = level.SaveVertices(vtx0.data(), vtx0.size(), attrs, + sizeof(RegionVertex)); + + std::vector idx(indices, indices + item.numIndices); + for (uint16 &i : idx) { + FByteswapper(i); + } + + glPrim.indices = level.SaveIndices(idx.data(), idx.size()).accessorIndex; + } +} + +void FoliageToGltf(const Foliage &foliage, IMGLTF &level, + const LevelIndexBuffer &idxBuffer, + const LevelVertexBuffer &vtxBuffer, + IGHWTOCIteratorConst instances, + std::map &materialRemaps, uint32 index) { + const uint16 *indexBuffer = &idxBuffer.data; + const char *vertexBuffer = &vtxBuffer.data; + + const uint16 *indices = indexBuffer + foliage.indexOffset; + const uint32 numVertices = + (foliage.spriteVertexOffset - foliage.branchVertexOffset) / + sizeof(BranchVertex); + + level.scenes.front().nodes.emplace_back(level.nodes.size()); + const size_t folNodeIndex = level.nodes.size(); + gltf::Node &glFoliageNode = level.nodes.emplace_back(); + glFoliageNode.name = "Foliage" + std::to_string(index); + + if (numVertices) { + const BranchVertex *vertices = reinterpret_cast( + vertexBuffer + foliage.branchVertexOffset); + std::vector vtx0(vertices, vertices + numVertices); + + glFoliageNode.mesh = level.meshes.size(); + gltf::Mesh &glMesh = level.meshes.emplace_back(); + + for (auto &v : vtx0) { + FByteswapper(v); + } + + AttributeUnormToSnorm sn; + + Attribute attrs[]{ + { + .type = uni::DataType::R16G16B16A16, + .format = uni::FormatType::FLOAT, + .usage = AttributeType::Position, + }, + { + .type = uni::DataType::R16G16, + .format = uni::FormatType::FLOAT, + .usage = AttributeType::TextureCoordiante, + }, + { + .type = uni::DataType::R8G8B8A8, + .format = uni::FormatType::UNORM, + .usage = AttributeType::Undefined, + }, + { + .type = uni::DataType::R8G8B8, + .format = uni::FormatType::UNORM, + .usage = AttributeType::Normal, + .customCodec = &sn, + }, + }; + + auto attrsa = level.SaveVertices(vtx0.data(), vtx0.size(), attrs, + sizeof(BranchVertex)); + + for (auto &r : foliage.branchLods) { + if (r.numIndices == 0) { + continue; + } + + gltf::Primitive &glPrim = glMesh.primitives.emplace_back(); + glPrim.attributes = attrsa; + + std::vector idx(indices + r.indexOffset, + indices + r.numIndices + r.indexOffset); + for (uint16 &i : idx) { + FByteswapper(i); + } + + glPrim.material = + materialRemaps + .try_emplace(foliage.textureIndex, + materialRemaps.size() + level.materials.size()) + .first->second; + + glPrim.indices = level.SaveIndices(idx.data(), idx.size()).accessorIndex; + + break; // only 1 lod + } + } + + std::vector tms; + + for (auto &inst : instances) { + if (inst.foliage == &foliage) { + tms.emplace_back(inst.tm); + } + } + + if (tms.size() == 1) { + memcpy(glFoliageNode.matrix.data(), tms.data(), 64); + } else if (tms.size() > 1) { + std::vector scales; + bool processScales = false; + + auto &str = level.GetTranslations(); + auto [accPos, accPosIndex] = level.NewAccessor(str, 4); + accPos.type = gltf::Accessor::Type::Vec3; + accPos.componentType = gltf::Accessor::ComponentType::Float; + accPos.count = tms.size(); + + auto [accRot, accRotIndex] = level.NewAccessor(str, 4, 12); + accRot.type = gltf::Accessor::Type::Vec4; + accRot.componentType = gltf::Accessor::ComponentType::Short; + accRot.normalized = true; + accRot.count = tms.size(); + Vector4A16::SetEpsilon(0.00001f); + + for (const es::Matrix44 &mtx : tms) { + uni::RTSValue val{}; + mtx.Decompose(val.translation, val.rotation, val.scale); + scales.emplace_back(val.scale); + + if (!processScales) { + processScales = val.scale != Vector4A16(1, 1, 1, 0); + } + + str.wr.Write(val.translation); + + val.rotation.Normalize() *= 0x7fff; + val.rotation = + Vector4A16(_mm_round_ps(val.rotation._data, _MM_ROUND_NEAREST)); + auto comp = val.rotation.Convert(); + str.wr.Write(comp); + } + + auto &attrs = + level.nodes.at(folNodeIndex) + .GetExtensionsAndExtras()["extensions"]["EXT_mesh_gpu_instancing"] + ["attributes"]; + + attrs["TRANSLATION"] = accPosIndex; + attrs["ROTATION"] = accRotIndex; + + if (processScales) { + auto &str = level.GetScales(); + auto [accScale, accScaleIndex] = level.NewAccessor(str, 4); + accScale.type = gltf::Accessor::Type::Vec3; + accScale.componentType = gltf::Accessor::ComponentType::Float; + accScale.count = tms.size(); + str.wr.WriteContainer(scales); + attrs["SCALE"] = accScaleIndex; + } + } + + for (uint32 i = 0; i < foliage.usedSpriteLods; i++) { + const SpriteLodRange &lod = foliage.spriteLodRanges[i]; + gltf::Mesh glMesh; + + for (uint32 s = 0; s < foliage.usedSpriteRanges; s++) { + const SpriteRange &r = foliage.spriteRanges[s]; + + struct SpriteVertexOut { + Vector position; + USVector2 uv; + uint32 unk; + }; + + std::vector idx(indices + r.indexBegin, indices + r.indexEnd); + uint16 numVertices = 0; + for (uint16 &i : idx) { + FByteswapper(i); + numVertices = std::max(i, numVertices); + } + numVertices++; + + const SpriteVertex *vertices = reinterpret_cast( + vertexBuffer + foliage.spriteVertexOffset); + std::vector inVerts(vertices, vertices + numVertices); + for (SpriteVertex &i : inVerts) { + FByteswapper(i); + } + std::vector outVerts; + + const Vector4 *centers = reinterpret_cast( + foliage.spritePositions.Get() + r.positionsOffset); + + for (uint32 i = 0; i < r.numSprites; i++) { + if (uint32 spriteIndex = i * 6 + r.indexBegin; + spriteIndex < lod.indexBegin || spriteIndex >= lod.indexEnd) { + continue; + } + + for (uint32 d = 0; d < 6; d++) { + uint32 vIndex = idx.at(i * 6 + d); + const SpriteVertex &vert = inVerts.at(vIndex); + SpriteVertexOut outVert{ + .position = Vector(centers[i]) + + Vector(vert.spriteSize[0], vert.spriteSize[1], 0), + .uv = vert.uv, + .unk = vert.unk, + }; + + outVerts.emplace_back(outVert); + } + + // possible todo: save center vertex (interpolate uvs, as TriangleFan + // [4, 0, 1, 2, 3]) + } + + Attribute attrs[]{ + { + .type = uni::DataType::R32G32B32, + .format = uni::FormatType::FLOAT, + .usage = AttributeType::Position, + }, + { + .type = uni::DataType::R16G16, + .format = uni::FormatType::FLOAT, + .usage = AttributeType::TextureCoordiante, + }, + }; + + if (outVerts.size() > 0) { + gltf::Primitive &glPrim = glMesh.primitives.emplace_back(); + glPrim.attributes = level.SaveVertices(outVerts.data(), outVerts.size(), + attrs, sizeof(SpriteVertexOut)); + glPrim.material = + materialRemaps + .try_emplace(foliage.textureIndex, + materialRemaps.size() + level.materials.size()) + .first->second; + } + } + + if (glMesh.primitives.size() > 0) { + level.nodes.at(folNodeIndex).children.emplace_back(level.nodes.size()); + gltf::Node &glNode = level.nodes.emplace_back(); + glNode.mesh = level.meshes.size(); + glNode.name = "Foliage" + std::to_string(index) + "_Sprites"; + glNode.GetExtensionsAndExtras() = + level.nodes.at(folNodeIndex).GetExtensionsAndExtras(); + level.meshes.emplace_back(std::move(glMesh)); + } + + break; // only 1 lod + } +} +/* +void ExtractTextures(AppContext *ctx, std::string path, + IGHWTOCIteratorConst textures, + std::istream &textureStream) { + for (uint16 index = 0; const Texture &tex : textures) { + TextureInfo info{ + .id = index++, + }; + ExtractTexture(ctx, path, tex, textureStream, info); + } +}*/ + +struct UnkInstance { + es::Matrix44 tm; + uint32 unk0[2]; + uint32 unk1; + uint32 unk2; +}; + +const int offset = (0x019C7626 - 0x01817080) / 2; + +// probably tesselated water +struct UnkEmbed : CoreClass { + static constexpr uint32 ID = 0xC650; + + uint32 unk0; + uint32 unkOffset; + uint32 numInstances; + uint32 instancesOffset; + uint32 unk4; + uint32 unk5; + uint32 null0[2]; + float unk6[3]; + uint32 unk7; + uint32 null1[20]; + uint32 unk8[12]; + + std::span Instances() { + return {reinterpret_cast(reinterpret_cast(this) + + instancesOffset), + numInstances}; + } +}; + +struct WaterIndex { + SVector face; + int16 adjIndices[5]; +}; + +struct WaterMesh { + float unk[2]; + uint32 numVertices; + uint32 numIndices; + Vector4A16 vertices[0x190]; + WaterIndex indices[0x31B]; +}; + +struct WaterMeshes : CoreClass { + static constexpr uint32 ID = 0x25600; + + Vector4A16 unk0; + uint32 numMeshes; + WaterMesh meshes[]; +}; + + +void FByteswapper(WaterMesh &item) { + FByteswapper(item.unk); + FByteswapper(item.numVertices); + FByteswapper(item.numIndices); + FArraySwapper(item.vertices); + FArraySwapper(item.indices); +} + +void FByteswapper(WaterMeshes &item) { + FByteswapper(item.unk0); + FByteswapper(item.numMeshes); + + for (uint32 i = 0; i < item.numMeshes; i++) { + FByteswapper(item.meshes[i]); + } +} + +void FByteswapper(UnkEmbed &item) { + FByteswapper(item.unk0); + FByteswapper(item.unkOffset); + FByteswapper(item.numInstances); + FByteswapper(item.instancesOffset); + FByteswapper(item.unk4); + FByteswapper(item.unk5); + FByteswapper(item.null0); + FByteswapper(item.unk6); + FByteswapper(item.unk7); + FByteswapper(item.null1); + FByteswapper(item.unk8); +} + +void FByteswapper(UnkInstance &item) { + FByteswapper(item.tm); + FByteswapper(item.unk0); + FByteswapper(item.unk1); + FByteswapper(item.unk2); +} + +void PlantsToGltf(const PlantPrimitive &prim, IMGLTF &level, + const LevelIndexBuffer &idxBuffer, + const LevelVertexBuffer &vtxBuffer, + // IGHWTOCIteratorConst instances, + std::map &materialRemaps, uint32 index) { + const uint16 *indices = &idxBuffer.data + prim.indexOffset; + std::vector idx(indices, indices + prim.numIndices); + for (uint16 &i : idx) { + FByteswapper(i); + } + + level.scenes.front().nodes.emplace_back(level.nodes.size()); + auto &glNode = level.nodes.emplace_back(); + glNode.mesh = level.meshes.size(); + glNode.name = "plant_" + std::to_string(index); + auto &glMesh = level.meshes.emplace_back(); + auto &glPrim = glMesh.primitives.emplace_back(); + auto outI = level.SaveIndices(idx.data(), idx.size()); + const uint32 numVertices = outI.maxIndex + 1; + glPrim.indices = outI.accessorIndex; + + glPrim.material = + materialRemaps.try_emplace(prim.materialIndex, materialRemaps.size()) + .first->second; + + const PlantVertex *vertices = reinterpret_cast( + &vtxBuffer.data + prim.vertexBufferOffset); + std::vector vtx0(vertices, vertices + numVertices); + + for (auto &v : vtx0) { + FByteswapper(v); + } + + Attribute attrs[]{ + { + .type = uni::DataType::R16G16B16A16, + .format = uni::FormatType::FLOAT, + .usage = AttributeType::Position, + }, + { + .type = uni::DataType::R8G8B8A8, + .format = uni::FormatType::UNORM, + .usage = AttributeType::VertexColor, + }, + { + .type = uni::DataType::R16G16, + .format = uni::FormatType::FLOAT, + .usage = AttributeType::TextureCoordiante, + }, + }; + + glPrim.attributes = + level.SaveVertices(vtx0.data(), vtx0.size(), attrs, sizeof(PlantVertex)); +} + +void AppProcessFile(AppContext *ctx) { + BinReaderRef_e rd(ctx->GetStream()); + IGHW main; + main.FromStream(rd, Version::RFOM); + + IGHWTOCIteratorConst mobys; + IGHWTOCIteratorConst ties; + IGHWTOCIteratorConst tieInstances; + IGHWTOCIteratorConst regionMeshes; + IGHWTOCIteratorConst lightmaps; + IGHWTOCIteratorConst textures; + IGHWTOCIteratorConst blendMaps; + IGHWTOCIteratorConst materials; + IGHWTOCIteratorConst shrubs; + IGHWTOCIteratorConst shrubInstances; + IGHWTOCIteratorConst foliages; + IGHWTOCIteratorConst foliageInstances; + + IGHWTOCIteratorConst unkEmbed; + IGHWTOCIteratorConst plantMeshes; + auto txStr = ctx->RequestFile("ps3leveltexs.dat"); + auto vtxStr = ctx->RequestFile("ps3levelverts.dat"); + IGHW buffers; + buffers.FromStream(*vtxStr.Get(), Version::RFOM); + IGHWTOCIteratorConst verts; + IGHWTOCIteratorConst indices; + CatchClasses(buffers, verts, indices); + CatchClasses(main, mobys, ties, tieInstances, regionMeshes, lightmaps, + textures, blendMaps, materials, shrubs, shrubInstances, foliages, + foliageInstances, unkEmbed, plantMeshes); + BinReaderRef_e txRd(*txStr.Get()); + txRd.SwapEndian(true); + + /*for (const MobyV1 &moby : mobys) { + MobyToGltf(moby, ctx, txRd, materials, textures.begin()); + }*/ + + IMGLTF level; + std::map materialRemaps; + std::map foliageRemaps; + + /*for (size_t tieIdx = 0; const TieV1 &tie : ties) { + TieToGltf(tie, level, indices.at(0), verts.at(0), tieInstances, tieIdx++, + materialRemaps); + } + + RegionToGltf(regionMeshes, level, indices.at(0), verts.at(0), materialRemaps); + + for (size_t tieIdx = 0; const Shrub &item : shrubs) { + ShrubToGltf(item, level, indices.at(0), verts.at(0), shrubInstances, + tieIdx++, materialRemaps); + }*/ + + for (size_t index = 0; auto &p : plantMeshes) { + PlantsToGltf(p, level, indices.at(0), verts.at(0), materialRemaps, index++); + } + + std::set textureRemaps; + MakeMaterials(ctx, level, materialRemaps, materials, textures.begin(), + txRd.BaseStream(), textureRemaps); + + /*for (size_t folIdx = 0; const Foliage &foliage : foliages) { + FoliageToGltf(foliage, level, indices.at(0), verts.at(0), foliageInstances, + foliageRemaps, folIdx++); + } + + MakeFoliageMaterials(ctx, level, foliageRemaps, textures.begin(), + txRd.BaseStream(), textureRemaps); + + UnkEmbed &unk = const_cast(unkEmbed.at(0)); + FByteswapper(unk); + auto instances = unk.Instances(); + + for (auto &i : instances) { + FByteswapper(i); + }*/ + + level.FinishAndSave( + ctx->NewFile(std::string(ctx->workingFile.GetFolder()) + "level.glb").str, + ""); + + /*ExtractTextures(ctx, "lightmap_", + reinterpret_cast &>(lightmaps), + txRd.BaseStream()); + ExtractTextures(ctx, "texture_", + reinterpret_cast &>(textures), + txRd.BaseStream()); + ExtractTextures(ctx, "blendmap_", + reinterpret_cast &>(blendMaps), + txRd.BaseStream());*/ +} diff --git a/main/CMakeLists.txt b/main/CMakeLists.txt new file mode 100644 index 0000000..c5e5e5a --- /dev/null +++ b/main/CMakeLists.txt @@ -0,0 +1,22 @@ +cmake_minimum_required(VERSION 3.12) + +project(MainToGLTF) + +build_target( + NAME + main_to_gltf + TYPE + ESMODULE + VERSION + 1 + SOURCES + extract.cpp + LINKS + gltf-interface + insomnia-interface + AUTHOR + "Lukas Cone" + DESCR + "Insomnia Engine levelmain extract" + START_YEAR + 2024) diff --git a/main/extract.cpp b/main/extract.cpp new file mode 100644 index 0000000..603b70e --- /dev/null +++ b/main/extract.cpp @@ -0,0 +1,1382 @@ +/* InsomniaToolset MainToGLTF + Copyright(C) 2024 Lukas Cone + + This program is free software : you can redistribute it and / or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program.If not, see . +*/ + +#include "insomnia/insomnia.hpp" +#include "insomnia/internal/vertex.hpp" +#include "nlohmann/json.hpp" +#include "project.h" +#include "spike/app_context.hpp" +#include "spike/except.hpp" +#include "spike/gltf.hpp" +#include "spike/io/binreader_stream.hpp" +#include "spike/master_printer.hpp" +#include "spike/reflect/reflector.hpp" +#include "spike/type/float.hpp" +#include "spike/uni/rts.hpp" +#include + +std::string_view filters[]{ + "^main.dat$", +}; + +static AppInfo_s appInfo{ + .header = MainToGLTF_DESC " v" MainToGLTF_VERSION ", " MainToGLTF_COPYRIGHT + "Lukas Cone", + .filters = filters, +}; + +AppInfo_s *AppInitModule() { return &appInfo; } + +struct IMGLTF : GLTFModel { + GLTFStream &GetTranslations() { + if (instTrs < 0) { + auto &str = NewStream("instance-tms", 20); + instTrs = str.slot; + return str; + } + + return Stream(instTrs); + } + + GLTFStream &GetScales() { + if (instScs < 0) { + auto &str = NewStream("instance-scale"); + instScs = str.slot; + return str; + } + + return Stream(instScs); + } + +private: + int32 instTrs = -1; + int32 instScs = -1; +}; + +struct Textures { + std::istream &textureStream; + std::istream *hiTextureStream = nullptr; + IGHWTOCIteratorConst high; + const Texture *textures; +}; + +struct TextureKey { + const Texture *tex; + const char *path = nullptr; + int32 id = -1; + bool albedo : 1 = false; + bool normal : 1 = false; + bool gloss : 1 = false; + bool specular : 1 = false; + bool emissive : 1 = false; + bool exponent : 1 = false; + + bool operator<(const TextureKey &o) const { + if (o.emissive == emissive) { + return tex < o.tex; + } + + return emissive < o.emissive; + } +}; + +struct TexStream : TexelOutput { + GLTF &main; + GLTFStream *str = nullptr; + bool ignore = false; + + TexStream(GLTF &main_) : main{main_} {} + + void SendData(std::string_view data) override { + if (str) [[likely]] { + str->wr.WriteContainer(data); + } + } + void NewFile(std::string path) override { + if (ignore) { + return; + } + + str = &main.NewStream(path); + } +}; + +void ExtractTexture(AppContext *ctx, std::string path, Textures tx, + TextureKey info, TexStream *texOut = nullptr) { + thread_local static std::string tmpBuffer(4 * 2048 * 2048, 0); + const uint32 txIndex = std::distance(tx.textures, info.tex); + + auto foundHi = tx.hiTextureStream + ? std::find_if(tx.high.begin(), tx.high.end(), + [txIndex](const HighmipTextureData &item) { + return item.textureIndex == txIndex; + }) + : tx.high.end(); + bool hasHigh = foundHi != tx.high.end(); + + if (hasHigh) { + tx.hiTextureStream->clear(); + tx.hiTextureStream->seekg(foundHi->dataOffset); + tx.hiTextureStream->read(tmpBuffer.data(), foundHi->dataSize * 0x80); + } else { + tx.textureStream.clear(); + tx.textureStream.seekg(info.tex->offset); + tx.textureStream.read(tmpBuffer.data(), tmpBuffer.size()); + } + + TexelTile tile = TexelTile::Linear; + + auto GetFormat = [&] { + using T = TextureFormat; + switch (info.tex->format) { + case T::RGBA8: + tile = TexelTile::Morton; + return TexelInputFormatType::RGBA8; + case T::R8: + tile = TexelTile::Morton; + return TexelInputFormatType::R8; + case T::BC1: + return TexelInputFormatType::BC1; + case T::BC3: + return TexelInputFormatType::BC3; + case T::BC2: + return TexelInputFormatType::BC2; + case T::R5G6B5: + tile = TexelTile::Morton; + return TexelInputFormatType::R5G6B5; + case T::RGBA4: + tile = TexelTile::Morton; + return TexelInputFormatType::RGBA4; + case T::RG8: + tile = TexelTile::Morton; + return TexelInputFormatType::RG8; + default: + PrintError("Invalid texture format: " + + std::to_string(int(info.tex->format))); + break; + } + + return TexelInputFormatType::INVALID; + }; + + TexelSwizzle swizzle; + + if (info.normal) { + swizzle.a = TexelSwizzleType::White; + swizzle.r = TexelSwizzleType::Alpha; + swizzle.g = TexelSwizzleType::Green; + swizzle.b = TexelSwizzleType::Black; + } else if (info.gloss) { + swizzle.r = swizzle.g = swizzle.b = TexelSwizzleType::RedInverted; + + if (info.specular) { + swizzle.a = TexelSwizzleType::Green; + } else { + swizzle.a = TexelSwizzleType::White; + } + } else if (info.emissive) { + swizzle.r = swizzle.g = swizzle.b = TexelSwizzleType::Blue; + swizzle.a = TexelSwizzleType::White; + } + + NewTexelContextCreate tctx{ + .width = uint16(info.tex->width * (hasHigh + 1)), + .height = uint16(info.tex->height * (hasHigh + 1)), + .baseFormat = + { + .type = GetFormat(), + .swizzle = swizzle, + .tile = tile, + .swapPacked = true, + }, + .depth = std::max( + uint16(1), uint16(info.tex->control3.Get())), + .numMipmaps = uint8(hasHigh ? 1 : info.tex->numMips), + .data = tmpBuffer.data(), + .texelOutput = texOut, + .formatOverride = + texOut ? TexelContextFormat::UPNG : TexelContextFormat::Config, + }; + + auto NormalPostProcess = [](char *data, uint32 stride, uint32 numTexels) { + for (uint32 i = 0; i < numTexels; i++, data += stride) { + UCVector *tx = reinterpret_cast(data); + Vector txf = tx->Convert(); + txf = (txf * (2.f / 0xff) - 1) * -1; + txf.x = 1; + txf.Normalize(); + txf.y *= -1; + *tx = ((txf + 1) * 0x7f).Convert(); + } + }; + + if (info.normal) { + tctx.postProcess = NormalPostProcess; + } + + auto EmissiveCheck = [texOut](char *data, uint32 stride, uint32 numTexels) { + texOut->ignore = true; + for (uint32 i = 0; i < numTexels; i++, data += stride) { + if (*data) { + texOut->ignore = false; + return; + } + } + }; + + if (info.emissive) { + tctx.postProcess = EmissiveCheck; + } + + if (texOut) { + ctx->NewImage(tctx); + } else { + ctx->ExtractContext()->NewImage(path + std::to_string(info.id), tctx); + } +} + +int32 TryExtractTexture(AppContext *ctx, GLTF &main, TextureKey key, + Textures tx, std::set &textureRemaps) { + if (auto found = textureRemaps.find(key); found != textureRemaps.end()) { + return found->id; + } + + gltf::Texture glTexture{}; + glTexture.source = main.textures.size(); + gltf::Image glImage{}; + glImage.mimeType = "image/png"; + + if (key.path) { + AFileInfo txInfo(key.path); + if (auto fileName = txInfo.GetFilename(); fileName.empty()) { + glImage.name = + "texture_" + std::to_string(std::distance(tx.textures, key.tex)); + } else { + glImage.name = fileName; + } + } else { + glImage.name = + "texture_" + std::to_string(std::distance(tx.textures, key.tex)); + } + + if (key.emissive) { + glImage.name.append("_e"); + } + + TexStream tStr{main}; + + ExtractTexture(ctx, glImage.name, tx, key, &tStr); + + if (!tStr.str) { + key.id = -1; + textureRemaps.emplace(key); + return -1; + } + + key.id = glTexture.source; + textureRemaps.emplace(key); + glImage.bufferView = tStr.str->slot; + main.textures.emplace_back(glTexture); + main.images.emplace_back(glImage); + return glTexture.source; +} + +void MakeMaterials( + AppContext *ctx, IMGLTF &main, + const std::map &materialRemaps, + IGHWTOCIteratorConst materials, + IGHWTOCIteratorConst materialPaths, + std::set &textureRemaps, Textures tx) { + main.materials.resize(materialRemaps.size()); + + for (auto [mid, mindex] : materialRemaps) { + gltf::Material &glMat = main.materials.at(mindex); + const MaterialResourceNameLookup &paths = materialPaths.at(mid); + glMat.name = paths.lookupPath.Get(); + glMat.pbrMetallicRoughness.metallicFactor = 0; + const MaterialV1_5 &mat = materials.at(mid); + const Texture *albedo = mat.diffuse; + + if (albedo) { + TextureKey albedoInfo{ + .tex = albedo, + .path = paths.mapLookupPaths[0], + .albedo = true, + }; + glMat.pbrMetallicRoughness.baseColorTexture.index = + TryExtractTexture(ctx, main, albedoInfo, tx, textureRemaps); + } + + if (mat.blendMode == 4) { + glMat.alphaMode = gltf::Material::AlphaMode::Mask; + } else if (mat.blendMode) { + glMat.alphaMode = gltf::Material::AlphaMode::Blend; + } + + const Texture *normal = mat.normal; + + if (normal) { + /*if (!mat.useNormalMap) { + PrintWarning("Material ", mid, " uses normal but no flags"); + }*/ + TextureKey normalInfo{ + .tex = normal, + .path = paths.mapLookupPaths[1], + .normal = true, + }; + + glMat.normalTexture.index = + TryExtractTexture(ctx, main, normalInfo, tx, textureRemaps); + } + + const Texture *special = mat.specular; + + if (special) { + if (!mat.useGlossiness && !mat.useSpecular) { + PrintWarning("Material ", mid, " uses special but no flags"); + } else { + TextureKey specInfo{ + .tex = special, + .gloss = mat.useGlossiness, + .specular = mat.useSpecular, + }; + + int32 specId = + TryExtractTexture(ctx, main, specInfo, tx, textureRemaps); + + if (mat.useSpecular) { + nlohmann::json &spec = + glMat.GetExtensionsAndExtras()["extensions"] + ["KHR_materials_specular"]; + spec["specularTexture"]["index"] = specId; + } + + if (mat.useGlossiness) { + glMat.pbrMetallicRoughness.metallicRoughnessTexture.index = specId; + } + } + + TextureKey emisKey{ + .tex = special, + .emissive = true, + }; + + glMat.emissiveTexture.index = + TryExtractTexture(ctx, main, emisKey, tx, textureRemaps); + } + } +} + +void MakeFoliageMaterials(AppContext *ctx, IMGLTF &main, + const std::map &materialRemaps, + Textures tx, std::set &textureRemaps) { + main.materials.resize(materialRemaps.size() + main.materials.size()); + + for (auto [mid, mindex] : materialRemaps) { + gltf::Material &glMat = main.materials.at(mindex); + glMat.name = "foliage_material_" + std::to_string(mid); + glMat.doubleSided = true; + const Texture &albedo = tx.textures[mid]; + TextureKey albedoInfo{ + .tex = &albedo, + .albedo = true, + }; + glMat.pbrMetallicRoughness.baseColorTexture.index = + TryExtractTexture(ctx, main, albedoInfo, tx, textureRemaps); + glMat.alphaMode = gltf::Material::AlphaMode::Mask; + } +} + +void MobyToGltf(const MobyV1 &moby, AppContext *ctx, + const LevelVertexBuffer &vtxBuffer, const char *path, + Textures tx, + IGHWTOCIteratorConst materialPaths, + IGHWTOCIteratorConst materials) { + IMGLTF main; + + const Skeleton *skeleton = moby.skeleton; + + for (uint32 i = 0; i < skeleton->numBones; i++) { + gltf::Node &glNode = main.nodes.emplace_back(); + glNode.name = std::to_string(i); + es::Matrix44 tm(skeleton->tms0[i]); + + if (int16 parentIndex = skeleton->bones[i].parentIndex; parentIndex < 0) { + main.scenes.front().nodes.emplace_back(i); + } else { + main.nodes.at(parentIndex).children.emplace_back(i); + es::Matrix44 ptm(skeleton->tms1[parentIndex]); + tm = ptm * tm; + } + + tm.r1().w = 0; + tm.r2().w = 0; + tm.r3().w = 0; + tm.r4().w = 1; + + memcpy(glNode.matrix.data(), &tm, 64); + } + + std::map joints; + const uint32 numMeshes = moby.numMeshes * (moby.anotherSet + 1); + + for (uint32 i = 0; i < numMeshes; i++) { + const Mesh &mesh = reinterpret_cast(moby.meshes[i]); + for (uint32 p = 0; p < mesh.numPrimitives; p++) { + const PrimitiveV2 &prim = mesh.primitives[p]; + + if (prim.numJoints < 2) { + // continue; + } + + for (uint32 j = 0; j < prim.numJoints; j++) { + uint16 joint = prim.joints[j]; + FByteswapper(joint); + joints.try_emplace(joint, joints.size()); + } + } + } + + { + gltf::Skin &skn = main.skins.emplace_back(); + skn.joints.resize(joints.size()); + GLTFStream &ibmStream = main.SkinStream(); + auto [acc, accId] = main.NewAccessor(ibmStream, 16); + acc.type = gltf::Accessor::Type::Mat4; + acc.componentType = gltf::Accessor::ComponentType::Float; + acc.count = joints.size(); + skn.inverseBindMatrices = accId; + std::vector ibms; + ibms.resize(joints.size()); + + for (auto &[jid, idx] : joints) { + skn.joints[idx] = jid; + es::Matrix44 ibm = skeleton->tms1[jid]; + ibm.r1().w = 0; + ibm.r2().w = 0; + ibm.r3().w = 0; + ibm.r4().w = 1; + ibms[idx] = ibm; + } + + ibmStream.wr.WriteContainer(ibms); + } + + struct AttributeBoneIndex : AttributeCodec { + AttributeBoneIndex(const std::map &joints_) + : joints(joints_) {} + void Sample(uni::FormatCodec::fvec &out, const char *input, + size_t stride) const override { + for (auto &o : out) { + int16 index = *reinterpret_cast(input); + uint16 joint = jointMap[std::abs((index + 1) / 3)]; + FByteswapper(joint); + o.x = joints.at(joint); + input += stride; + } + } + void Transform(uni::FormatCodec::fvec &) const override {} + bool CanSample() const override { return true; } + bool CanTransform() const override { return false; } + bool IsNormalized() const override { return false; } + + const std::map &joints; + const uint16 *jointMap = nullptr; + } attributeBoneIndex{joints}; + + struct AttributeBoneIndices : AttributeCodec { + AttributeBoneIndices(const std::map &joints_) + : joints(joints_) {} + void Sample(uni::FormatCodec::fvec &out, const char *input, + size_t stride) const override { + for (auto &o : out) { + UCVector4 index = *reinterpret_cast(input); + for (uint32 i = 0; i < 4; i++) { + uint16 joint = jointMap[index[i]]; + FByteswapper(joint); + o[i] = joints.at(joint); + } + input += stride; + } + } + void Transform(uni::FormatCodec::fvec &) const override {} + bool CanSample() const override { return true; } + bool CanTransform() const override { return false; } + bool IsNormalized() const override { return false; } + + const std::map &joints; + const uint16 *jointMap = nullptr; + } attributeBoneIndices{joints}; + + struct AttributeMul : AttributeCodec { + AttributeMul(float scale) : mul(scale * 0x7fff) {} + void Sample(uni::FormatCodec::fvec &, const char *, size_t) const override { + } + void Transform(uni::FormatCodec::fvec &in) const override { + for (Vector4A16 &v : in) { + v = v * mul; + } + } + bool CanSample() const override { return false; } + bool CanTransform() const override { return true; } + bool IsNormalized() const override { return false; } + + Vector4A16 mul; + } attributeMul{moby.meshScale}; + + std::map materialRemaps; + BinReaderRef_e stream(tx.textureStream); + stream.SwapEndian(true); + + for (uint32 i = 0; i < numMeshes; i++) { + const Mesh &mesh = reinterpret_cast(moby.meshes[i]); + if (mesh.numPrimitives == 0 || !mesh.primitives) { + continue; + } + + main.scenes.front().nodes.emplace_back(main.nodes.size()); + gltf::Node &glNode = main.nodes.emplace_back(); + glNode.mesh = main.meshes.size(); + glNode.skin = 0; + glNode.name = "Mesh_" + std::to_string(i); + gltf::Mesh &glMesh = main.meshes.emplace_back(); + + for (uint32 p = 0; p < mesh.numPrimitives; p++) { + const PrimitiveV2 &prim = mesh.primitives[p]; + gltf::Primitive &glPrim = glMesh.primitives.emplace_back(); + glPrim.material = + materialRemaps.try_emplace(prim.materialIndex, materialRemaps.size()) + .first->second; + + if (moby.vertexBufferOffset >= 0) { + stream.Seek(moby.vertexBufferOffset + prim.vertexOffset); + } + + if (prim.vertexFormat == 0) { + std::vector vtx0; + + if (moby.vertexBufferOffset >= 0) { + stream.ReadContainer(vtx0, prim.numVertices); + } else { + vtx0.reserve(prim.numVertices); + const Vertex0 *vtStart = reinterpret_cast( + &vtxBuffer.data + (moby.vertexBufferOffset & 0x7fffffff) + + prim.vertexOffset); + vtx0.insert(vtx0.begin(), vtStart, vtStart + prim.numVertices); + + for (auto &v : vtx0) { + FByteswapper(v); + } + } + + attributeBoneIndex.jointMap = prim.joints; + + Attribute attrs[]{ + { + .type = uni::DataType::R16G16B16, + .format = uni::FormatType::NORM, + .usage = AttributeType::Position, + .customCodec = &attributeMul, + }, + { + .type = uni::DataType::R16, + .format = uni::FormatType::INVALID, + .usage = AttributeType::BoneIndices, + .customCodec = &attributeBoneIndex, + }, + { + .type = uni::DataType::R16G16, + .format = uni::FormatType::FLOAT, + .usage = AttributeType::TextureCoordiante, + }, + { + .type = uni::DataType::R11G11B10, + .format = uni::FormatType::NORM, + .usage = AttributeType::Normal, + }, + }; + + glPrim.attributes = + main.SaveVertices(vtx0.data(), vtx0.size(), attrs, sizeof(Vertex0)); + } else { + std::vector vtx1; + if (moby.vertexBufferOffset >= 0) { + stream.ReadContainer(vtx1, prim.numVertices); + } else { + vtx1.reserve(prim.numVertices); + const Vertex1 *vtStart = reinterpret_cast( + &vtxBuffer.data + (moby.vertexBufferOffset & 0x7fffffff) + + prim.vertexOffset); + vtx1.insert(vtx1.begin(), vtStart, vtStart + prim.numVertices); + + for (auto &v : vtx1) { + FByteswapper(v); + } + } + + attributeBoneIndices.jointMap = prim.joints; + + Attribute attrs[]{ + { + .type = uni::DataType::R16G16B16A16, + .format = uni::FormatType::NORM, + .usage = AttributeType::Position, + .customCodec = &attributeMul, + }, + { + .type = uni::DataType::R8G8B8A8, + .format = uni::FormatType::INVALID, + .usage = AttributeType::BoneIndices, + .customCodec = &attributeBoneIndices, + }, + { + .type = uni::DataType::R8G8B8A8, + .format = uni::FormatType::UNORM, + .usage = AttributeType::BoneWeights, + }, + { + .type = uni::DataType::R16G16, + .format = uni::FormatType::FLOAT, + .usage = AttributeType::TextureCoordiante, + }, + { + .type = uni::DataType::R11G11B10, + .format = uni::FormatType::NORM, + .usage = AttributeType::Normal, + }, + }; + + glPrim.attributes = + main.SaveVertices(vtx1.data(), vtx1.size(), attrs, sizeof(Vertex1)); + } + + stream.Seek(moby.indexBufferOffset + prim.indexOffset * 2); + + std::vector idx; + stream.ReadContainer(idx, prim.numIndices); + + glPrim.indices = main.SaveIndices(idx.data(), idx.size()).accessorIndex; + } + } + + std::set textureRemaps; + MakeMaterials(ctx, main, materialRemaps, materials, materialPaths, + textureRemaps, tx); + + main.FinishAndSave( + ctx->NewFile(std::string(ctx->workingFile.GetFolder()) + path + ".glb") + .str, + ""); +} + +void TieToGltf(const TieV1_5 &tie, IMGLTF &level, + const LevelIndexBuffer &idxBuffer, + const LevelVertexBuffer &vtxBuffer, + IGHWTOCIteratorConst tieInstances, + const char *path, std::map &materialRemaps) { + const uint16 *indexBuffer = &idxBuffer.data; + const char *vertexBuffer = &vtxBuffer.data; + + level.scenes.front().nodes.emplace_back(level.nodes.size()); + gltf::Node &glNode = level.nodes.emplace_back(); + glNode.mesh = level.meshes.size(); + AFileInfo tinf(path); + glNode.name = tinf.GetFilename(); + gltf::Mesh &glMesh = level.meshes.emplace_back(); + + struct AttributeMul : AttributeCodec { + AttributeMul(Vector scale) : mul(scale * 0x7fff) {} + void Sample(uni::FormatCodec::fvec &, const char *, size_t) const override { + } + void Transform(uni::FormatCodec::fvec &in) const override { + for (Vector4A16 &v : in) { + v = v * mul; + } + } + bool CanSample() const override { return false; } + bool CanTransform() const override { return true; } + bool IsNormalized() const override { return false; } + + Vector4A16 mul; + } attributeMul{tie.meshScale}; + + for (uint32 p = 0; p < tie.numMeshes; p++) { + const TiePrimitiveV2 &prim = tie.primitives[p]; + gltf::Primitive &glPrim = glMesh.primitives.emplace_back(); + glPrim.material = + materialRemaps.try_emplace(prim.materialIndex, materialRemaps.size()) + .first->second; + + const uint16 *indices = indexBuffer + prim.indexOffset; + const Vertex0 *vertices = reinterpret_cast( + vertexBuffer + tie.vertexBufferOffset0) + + prim.vertexOffset0; + + std::vector vtx0(vertices, vertices + prim.numVertices); + + for (auto &v : vtx0) { + FByteswapper(v); + } + + Attribute attrs[]{ + { + .type = uni::DataType::R16G16B16A16, + .format = uni::FormatType::NORM, + .usage = AttributeType::Position, + .customCodec = &attributeMul, + }, + { + .type = uni::DataType::R16G16, + .format = uni::FormatType::FLOAT, + .usage = AttributeType::TextureCoordiante, + }, + { + .type = uni::DataType::R11G11B10, + .format = uni::FormatType::NORM, + .usage = AttributeType::Normal, + }, + }; + + glPrim.attributes = + level.SaveVertices(vtx0.data(), vtx0.size(), attrs, sizeof(Vertex0)); + + std::vector idx(indices, indices + prim.numIndices); + for (uint16 &i : idx) { + FByteswapper(i); + } + + glPrim.indices = level.SaveIndices(idx.data(), idx.size()).accessorIndex; + } + + std::vector tms; + + for (auto &inst : tieInstances) { + if (inst.tie == &tie) { + tms.emplace_back(inst.tm); + } + } + + if (tms.size() == 1) { + memcpy(glNode.matrix.data(), tms.data(), 64); + } else if (tms.size() > 1) { + std::vector scales; + bool processScales = false; + + auto &str = level.GetTranslations(); + auto [accPos, accPosIndex] = level.NewAccessor(str, 4); + accPos.type = gltf::Accessor::Type::Vec3; + accPos.componentType = gltf::Accessor::ComponentType::Float; + accPos.count = tms.size(); + + auto [accRot, accRotIndex] = level.NewAccessor(str, 4, 12); + accRot.type = gltf::Accessor::Type::Vec4; + accRot.componentType = gltf::Accessor::ComponentType::Short; + accRot.normalized = true; + accRot.count = tms.size(); + Vector4A16::SetEpsilon(0.00001f); + + for (const es::Matrix44 &mtx : tms) { + uni::RTSValue val{}; + mtx.Decompose(val.translation, val.rotation, val.scale); + scales.emplace_back(val.scale); + + if (!processScales) { + processScales = val.scale != Vector4A16(1, 1, 1, 0); + } + + str.wr.Write(val.translation); + + val.rotation.Normalize() *= 0x7fff; + val.rotation = + Vector4A16(_mm_round_ps(val.rotation._data, _MM_ROUND_NEAREST)); + auto comp = val.rotation.Convert(); + str.wr.Write(comp); + } + + auto &attrs = + glNode.GetExtensionsAndExtras()["extensions"]["EXT_mesh_gpu_instancing"] + ["attributes"]; + + attrs["TRANSLATION"] = accPosIndex; + attrs["ROTATION"] = accRotIndex; + + if (processScales) { + auto &str = level.GetScales(); + auto [accScale, accScaleIndex] = level.NewAccessor(str, 4); + accScale.type = gltf::Accessor::Type::Vec3; + accScale.componentType = gltf::Accessor::ComponentType::Float; + accScale.count = tms.size(); + str.wr.WriteContainer(scales); + attrs["SCALE"] = accScaleIndex; + } + } +} + +void ShrubToGltf(const Shrub &shrub, IMGLTF &level, + const LevelIndexBuffer &idxBuffer, + const LevelVertexBuffer &vtxBuffer, + IGHWTOCIteratorConst shrubInstances, + uint32 index, std::map &materialRemaps) { + const uint16 *indexBuffer = &idxBuffer.data; + const char *vertexBuffer = &vtxBuffer.data; + + level.scenes.front().nodes.emplace_back(level.nodes.size()); + gltf::Node &glNode = level.nodes.emplace_back(); + glNode.mesh = level.meshes.size(); + glNode.name = "TieMesh_" + std::to_string(index); + gltf::Mesh &glMesh = level.meshes.emplace_back(); + + struct AttributeMul : AttributeCodec { + AttributeMul(Vector scale) : mul(scale) {} + void Sample(uni::FormatCodec::fvec &, const char *, size_t) const override { + } + void Transform(uni::FormatCodec::fvec &in) const override { + for (Vector4A16 &v : in) { + v = v * mul; + } + } + bool CanSample() const override { return false; } + bool CanTransform() const override { return true; } + bool IsNormalized() const override { return false; } + + Vector4A16 mul; + } attributeMul{shrub.meshScale / 0x1000}; + + for (uint32 p = 0; p < shrub.numPrimitives; p++) { + const ShrubPrimitive &prim = shrub.primitives[p]; + gltf::Primitive &glPrim = glMesh.primitives.emplace_back(); + glPrim.material = + materialRemaps.try_emplace(prim.materialIndex, materialRemaps.size()) + .first->second; + + const uint16 *indices = indexBuffer + prim.indexOffset; + const Vertex0 *vertices = reinterpret_cast( + vertexBuffer + prim.vertexBufferOffset); + + std::vector vtx0(vertices, vertices + prim.numVertices); + + for (auto &v : vtx0) { + FByteswapper(v); + } + + Attribute attrs[]{ + { + .type = uni::DataType::R16G16B16A16, + .format = uni::FormatType::NORM, + .usage = AttributeType::Position, + .customCodec = &attributeMul, + }, + { + .type = uni::DataType::R16G16, + .format = uni::FormatType::FLOAT, + .usage = AttributeType::TextureCoordiante, + }, + { + .type = uni::DataType::R11G11B10, + .format = uni::FormatType::NORM, + .usage = AttributeType::Normal, + }, + }; + + glPrim.attributes = + level.SaveVertices(vtx0.data(), vtx0.size(), attrs, sizeof(Vertex0)); + + std::vector idx(indices, indices + prim.numIndices); + for (uint16 &i : idx) { + FByteswapper(i); + } + + glPrim.indices = level.SaveIndices(idx.data(), idx.size()).accessorIndex; + } + + std::vector tms; + + for (auto &inst : shrubInstances) { + if (inst.shrub == &shrub) { + tms.emplace_back(inst.tm); + } + } + + if (tms.size() == 1) { + memcpy(glNode.matrix.data(), tms.data(), 64); + } else if (tms.size() > 1) { + std::vector scales; + bool processScales = false; + + auto &str = level.GetTranslations(); + auto [accPos, accPosIndex] = level.NewAccessor(str, 4); + accPos.type = gltf::Accessor::Type::Vec3; + accPos.componentType = gltf::Accessor::ComponentType::Float; + accPos.count = tms.size(); + + auto [accRot, accRotIndex] = level.NewAccessor(str, 4, 12); + accRot.type = gltf::Accessor::Type::Vec4; + accRot.componentType = gltf::Accessor::ComponentType::Short; + accRot.normalized = true; + accRot.count = tms.size(); + Vector4A16::SetEpsilon(0.00001f); + + for (const es::Matrix44 &mtx : tms) { + uni::RTSValue val{}; + mtx.Decompose(val.translation, val.rotation, val.scale); + scales.emplace_back(val.scale); + + if (!processScales) { + processScales = val.scale != Vector4A16(1, 1, 1, 0); + } + + str.wr.Write(val.translation); + + val.rotation.Normalize() *= 0x7fff; + val.rotation = + Vector4A16(_mm_round_ps(val.rotation._data, _MM_ROUND_NEAREST)); + auto comp = val.rotation.Convert(); + str.wr.Write(comp); + } + + auto &attrs = + glNode.GetExtensionsAndExtras()["extensions"]["EXT_mesh_gpu_instancing"] + ["attributes"]; + + attrs["TRANSLATION"] = accPosIndex; + attrs["ROTATION"] = accRotIndex; + + if (processScales) { + auto &str = level.GetScales(); + auto [accScale, accScaleIndex] = level.NewAccessor(str, 4); + accScale.type = gltf::Accessor::Type::Vec3; + accScale.componentType = gltf::Accessor::ComponentType::Float; + accScale.count = tms.size(); + str.wr.WriteContainer(scales); + attrs["SCALE"] = accScaleIndex; + } + } +} + +void RegionToGltf(IGHWTOCIteratorConst items, IMGLTF &level, + const LevelIndexBuffer &idxBuffer, + const LevelVertexBuffer &vtxBuffer, + std::map &materialRemaps) { + const uint16 *indexBuffer = &idxBuffer.data; + const char *vertexBuffer = &vtxBuffer.data; + + level.scenes.front().nodes.emplace_back(level.nodes.size()); + gltf::Node &glNode = level.nodes.emplace_back(); + glNode.mesh = level.meshes.size(); + glNode.name = "RegionMesh"; + gltf::Mesh &glMesh = level.meshes.emplace_back(); + + for (const RegionMeshV2 &item : items) { + gltf::Primitive &glPrim = glMesh.primitives.emplace_back(); + glPrim.material = + materialRemaps.try_emplace(item.materialIndex, materialRemaps.size()) + .first->second; + + const uint16 *indices = indexBuffer + item.indexOffset; + const RegionVertexV2 *vertices = reinterpret_cast( + vertexBuffer + item.vertexOffset); + + std::vector vtx0(vertices, vertices + item.numVerties); + + for (auto &v : vtx0) { + FByteswapper(v); + } + + AttributeMad attributeMad; + attributeMad.mul = Vector4A16(0x7fff) / 0x100; + attributeMad.add = item.position / 0x100; + + Attribute attrs[]{ + { + .type = uni::DataType::R16G16B16A16, + .format = uni::FormatType::NORM, + .usage = AttributeType::Position, + .customCodec = &attributeMad, + }, + { + .type = uni::DataType::R16G16, + .format = uni::FormatType::FLOAT, + .usage = AttributeType::TextureCoordiante, + }, + { + .type = uni::DataType::R16G16, + .format = uni::FormatType::FLOAT, + .usage = AttributeType::TextureCoordiante, + }, + { + .type = uni::DataType::R11G11B10, + .format = uni::FormatType::NORM, + .usage = AttributeType::Normal, + }, + }; + + glPrim.attributes = level.SaveVertices(vtx0.data(), vtx0.size(), attrs, + sizeof(RegionVertexV2)); + + std::vector idx(indices, indices + item.numIndices); + for (uint16 &i : idx) { + FByteswapper(i); + } + + glPrim.indices = level.SaveIndices(idx.data(), idx.size()).accessorIndex; + } +} + +void FoliageToGltf(const Foliage &foliage, IMGLTF &level, + const LevelIndexBuffer &idxBuffer, + const LevelVertexBuffer &vtxBuffer, + IGHWTOCIteratorConst instances, + std::map &materialRemaps, uint32 index) { + const uint16 *indexBuffer = &idxBuffer.data; + const char *vertexBuffer = &vtxBuffer.data; + + const uint16 *indices = indexBuffer + foliage.indexOffset; + const uint32 numVertices = + (foliage.spriteVertexOffset - foliage.branchVertexOffset) / + sizeof(BranchVertex); + + level.scenes.front().nodes.emplace_back(level.nodes.size()); + const size_t folNodeIndex = level.nodes.size(); + gltf::Node &glFoliageNode = level.nodes.emplace_back(); + glFoliageNode.name = "Foliage" + std::to_string(index); + + if (numVertices) { + const BranchVertex *vertices = reinterpret_cast( + vertexBuffer + foliage.branchVertexOffset); + std::vector vtx0(vertices, vertices + numVertices); + + glFoliageNode.mesh = level.meshes.size(); + gltf::Mesh &glMesh = level.meshes.emplace_back(); + + for (auto &v : vtx0) { + FByteswapper(v); + } + + AttributeUnormToSnorm sn; + + Attribute attrs[]{ + { + .type = uni::DataType::R16G16B16A16, + .format = uni::FormatType::FLOAT, + .usage = AttributeType::Position, + }, + { + .type = uni::DataType::R16G16, + .format = uni::FormatType::FLOAT, + .usage = AttributeType::TextureCoordiante, + }, + { + .type = uni::DataType::R8G8B8A8, + .format = uni::FormatType::UNORM, + .usage = AttributeType::Undefined, + }, + { + .type = uni::DataType::R8G8B8, + .format = uni::FormatType::UNORM, + .usage = AttributeType::Normal, + .customCodec = &sn, + }, + }; + + auto attrsa = level.SaveVertices(vtx0.data(), vtx0.size(), attrs, + sizeof(BranchVertex)); + + for (auto &r : foliage.branchLods) { + if (r.numIndices == 0) { + continue; + } + + gltf::Primitive &glPrim = glMesh.primitives.emplace_back(); + glPrim.attributes = attrsa; + + std::vector idx(indices + r.indexOffset, + indices + r.numIndices + r.indexOffset); + for (uint16 &i : idx) { + FByteswapper(i); + } + + glPrim.material = + materialRemaps + .try_emplace(foliage.textureIndex, + materialRemaps.size() + level.materials.size()) + .first->second; + + glPrim.indices = level.SaveIndices(idx.data(), idx.size()).accessorIndex; + + break; // only 1 lod + } + } + + std::vector tms; + + for (auto &inst : instances) { + if (inst.foliage == &foliage) { + tms.emplace_back(inst.tm); + } + } + + if (tms.size() == 1) { + memcpy(glFoliageNode.matrix.data(), tms.data(), 64); + } else if (tms.size() > 1) { + std::vector scales; + bool processScales = false; + + auto &str = level.GetTranslations(); + auto [accPos, accPosIndex] = level.NewAccessor(str, 4); + accPos.type = gltf::Accessor::Type::Vec3; + accPos.componentType = gltf::Accessor::ComponentType::Float; + accPos.count = tms.size(); + + auto [accRot, accRotIndex] = level.NewAccessor(str, 4, 12); + accRot.type = gltf::Accessor::Type::Vec4; + accRot.componentType = gltf::Accessor::ComponentType::Short; + accRot.normalized = true; + accRot.count = tms.size(); + Vector4A16::SetEpsilon(0.00001f); + + for (const es::Matrix44 &mtx : tms) { + uni::RTSValue val{}; + mtx.Decompose(val.translation, val.rotation, val.scale); + scales.emplace_back(val.scale); + + if (!processScales) { + processScales = val.scale != Vector4A16(1, 1, 1, 0); + } + + str.wr.Write(val.translation); + + val.rotation.Normalize() *= 0x7fff; + val.rotation = + Vector4A16(_mm_round_ps(val.rotation._data, _MM_ROUND_NEAREST)); + auto comp = val.rotation.Convert(); + str.wr.Write(comp); + } + + auto &attrs = + level.nodes.at(folNodeIndex) + .GetExtensionsAndExtras()["extensions"]["EXT_mesh_gpu_instancing"] + ["attributes"]; + + attrs["TRANSLATION"] = accPosIndex; + attrs["ROTATION"] = accRotIndex; + + if (processScales) { + auto &str = level.GetScales(); + auto [accScale, accScaleIndex] = level.NewAccessor(str, 4); + accScale.type = gltf::Accessor::Type::Vec3; + accScale.componentType = gltf::Accessor::ComponentType::Float; + accScale.count = tms.size(); + str.wr.WriteContainer(scales); + attrs["SCALE"] = accScaleIndex; + } + } + + for (uint32 i = 0; i < foliage.usedSpriteLods; i++) { + const SpriteLodRange &lod = foliage.spriteLodRanges[i]; + gltf::Mesh glMesh; + + for (uint32 s = 0; s < foliage.usedSpriteRanges; s++) { + const SpriteRange &r = foliage.spriteRanges[s]; + + struct SpriteVertexOut { + Vector position; + USVector2 uv; + uint32 unk; + }; + + std::vector idx(indices + r.indexBegin, indices + r.indexEnd); + uint16 numVertices = 0; + for (uint16 &i : idx) { + FByteswapper(i); + numVertices = std::max(i, numVertices); + } + numVertices++; + + const SpriteVertex *vertices = reinterpret_cast( + vertexBuffer + foliage.spriteVertexOffset); + std::vector inVerts(vertices, vertices + numVertices); + for (SpriteVertex &i : inVerts) { + FByteswapper(i); + } + std::vector outVerts; + + const Vector4 *centers = reinterpret_cast( + foliage.spritePositions.Get() + r.positionsOffset); + + for (uint32 i = 0; i < r.numSprites; i++) { + if (uint32 spriteIndex = i * 6 + r.indexBegin; + spriteIndex < lod.indexBegin || spriteIndex >= lod.indexEnd) { + continue; + } + + for (uint32 d = 0; d < 6; d++) { + uint32 vIndex = idx.at(i * 6 + d); + const SpriteVertex &vert = inVerts.at(vIndex); + SpriteVertexOut outVert{ + .position = Vector(centers[i]) + + Vector(vert.spriteSize[0], vert.spriteSize[1], 0), + .uv = vert.uv, + .unk = vert.unk, + }; + + outVerts.emplace_back(outVert); + } + + // possible todo: save center vertex (interpolate uvs, as TriangleFan + // [4, 0, 1, 2, 3]) + } + + Attribute attrs[]{ + { + .type = uni::DataType::R32G32B32, + .format = uni::FormatType::FLOAT, + .usage = AttributeType::Position, + }, + { + .type = uni::DataType::R16G16, + .format = uni::FormatType::FLOAT, + .usage = AttributeType::TextureCoordiante, + }, + }; + + if (outVerts.size() > 0) { + gltf::Primitive &glPrim = glMesh.primitives.emplace_back(); + glPrim.attributes = level.SaveVertices(outVerts.data(), outVerts.size(), + attrs, sizeof(SpriteVertexOut)); + glPrim.material = + materialRemaps + .try_emplace(foliage.textureIndex, + materialRemaps.size() + level.materials.size()) + .first->second; + } + } + + if (glMesh.primitives.size() > 0) { + level.nodes.at(folNodeIndex).children.emplace_back(level.nodes.size()); + gltf::Node &glNode = level.nodes.emplace_back(); + glNode.mesh = level.meshes.size(); + glNode.name = "Foliage" + std::to_string(index) + "_Sprites"; + glNode.GetExtensionsAndExtras() = + level.nodes.at(folNodeIndex).GetExtensionsAndExtras(); + level.meshes.emplace_back(std::move(glMesh)); + } + + break; // only 1 lod + } +} +/* +void ExtractTextures(AppContext *ctx, std::string path, + IGHWTOCIteratorConst textures, Textures tx) { + for (uint16 index = 0; const Texture &tex : textures) { + TextureInfo info{ + .id = index++, + }; + + ExtractTexture(ctx, path, tex, tx, info); + } +}*/ + +void AppProcessFile(AppContext *ctx) { + BinReaderRef_e rd(ctx->GetStream()); + IGHW main; + main.FromStream(rd, Version::TOD); + + IGHWTOCIteratorConst lightmaps; + IGHWTOCIteratorConst textures; + IGHWTOCIteratorConst blendMaps; + IGHWTOCIteratorConst highTextures; + IGHWTOCIteratorConst mobys; + IGHWTOCIteratorConst materials; + IGHWTOCIteratorConst ties; + IGHWTOCIteratorConst tieInstances; + + IGHWTOCIteratorConst regionMeshes; + + IGHWTOCIteratorConst mobyPaths; + IGHWTOCIteratorConst materialPaths; + IGHWTOCIteratorConst tiePaths; + + auto txStr = ctx->RequestFile("textures.dat"); + auto txsStr = ctx->RequestFile("texstream.dat"); + auto vtxStr = ctx->RequestFile("vertices.dat"); + auto dbgStr = ctx->RequestFile("debug.dat"); + IGHW debug; + debug.FromStream(*dbgStr.Get(), Version::TOD); + CatchClasses(debug, mobyPaths, materialPaths, tiePaths); + IGHW buffers; + buffers.FromStream(*vtxStr.Get(), Version::TOD); + IGHWTOCIteratorConst verts; + IGHWTOCIteratorConst indices; + CatchClasses(buffers, verts, indices); + CatchClasses(main, lightmaps, textures, blendMaps, highTextures, mobys, + materials, ties, tieInstances, regionMeshes); + BinReaderRef_e txRd(*txStr.Get()); + txRd.SwapEndian(true); + + Textures tx{ + .textureStream = txRd.BaseStream(), + .hiTextureStream = txsStr.Get(), + .high = highTextures, + .textures = textures.begin(), + }; + + /*for (uint32 index = 0; const MobyV1 &moby : mobys) { + const char *path = mobyPaths.at(index++).path; + MobyToGltf(moby, ctx, verts.at(0), path, tx, materialPaths, materials); + }*/ + + IMGLTF level; + std::map materialRemaps; + std::map foliageRemaps; + + for (size_t tieIdx = 0; const TieV1_5 &tie : ties) { + const char *path = tiePaths.at(tieIdx++).path; + TieToGltf(tie, level, indices.at(0), verts.at(0), tieInstances, path, + materialRemaps); + } + + RegionToGltf(regionMeshes, level, indices.at(0), verts.at(0), materialRemaps); + + /*for (size_t tieIdx = 0; const Shrub &item : shrubs) { + ShrubToGltf(item, level, indices.at(0), verts.at(0), shrubInstances, + tieIdx++, materialRemaps); + }*/ + + std::set textureRemaps; + MakeMaterials(ctx, level, materialRemaps, materials, materialPaths, + textureRemaps, tx); + + /*for (size_t folIdx = 0; const Foliage &foliage : foliages) { + FoliageToGltf(foliage, level, indices.at(0), verts.at(0), foliageInstances, + foliageRemaps, folIdx++); + }*/ + + // MakeFoliageMaterials(ctx, level, foliageRemaps, tx, textureRemaps); + + level.FinishAndSave( + ctx->NewFile(std::string(ctx->workingFile.GetFolder()) + "level.glb").str, + ""); + + /* ExtractTextures(ctx, "lightmap_", + reinterpret_cast &>(lightmaps), + highTextures, txRd.BaseStream()); + ExtractTextures(ctx, "texture_", textures, highTextures, txRd.BaseStream(), + txsStr.Get()); + ExtractTextures(ctx, "shadowmap_", + reinterpret_cast &>(blendMaps), + highTextures, txRd.BaseStream());*/ +}