Async resource loading refactor & Terrain improvements (#223)

code/addons/dynui/imguicontext.cc

@@ -180,7 +180,7 @@ ImguiDrawFunction(const CoreGraphics::CmdBufferId cmdBuf, const Math::rectangle<
 
                 TextureInfo texInfo;
                 texInfo.type = 0;
-                texInfo.useAlpha = 1;
+                texInfo.useAlpha = tex.useAlpha;
 
                 // set texture in shader, we shouldn't have to put it into ImGui
                 CoreGraphics::TextureId texture = tex.nebulaHandle;
@@ -600,6 +600,7 @@ ImguiContext::Create()
     state.fontTexture.nebulaHandle = CoreGraphics::CreateTexture(texInfo);
     state.fontTexture.mip = 0;
     state.fontTexture.layer = 0;
+    state.fontTexture.useAlpha = true;
     io.Fonts->TexID = &state.fontTexture;
     io.Fonts->ClearTexData();
 

code/addons/dynui/imguicontext.h

@@ -28,8 +28,9 @@ namespace Dynui
 struct ImguiTextureId
 {
     CoreGraphics::TextureId nebulaHandle;
-    uint8 layer = 0;
-    uint8 mip = 0;
+    uint layer : 8 = 0;
+    uint mip : 4 = 0;
+    uint useAlpha : 1 = 0;
 };
 
 class ImguiContext : public Graphics::GraphicsContext

code/addons/graphicsfeature/graphicsfeatureunit.cc

@@ -180,7 +180,6 @@ GraphicsFeatureUnit::OnActivate()
         Terrain::TerrainContext::Create(settings);
         Terrain::TerrainContext::SetSun(this->globalLight);
 
-        /*
         this->terrain.entity = Graphics::CreateEntity();
         Graphics::RegisterEntity<Terrain::TerrainContext>(this->terrain.entity);
         Terrain::TerrainContext::SetupTerrain(this->terrain.entity, terrainSettings.instance->height, terrainSettings.instance->decision, terrainSettings.config->raytracing);
@@ -236,7 +235,6 @@ GraphicsFeatureUnit::OnActivate()
                 Math::vec2 {terrainSettings.config->world_size_width, terrainSettings.config->world_size_height}};
             Vegetation::VegetationContext::Create(vegSettings);
         }
-        */
     }
 
     Lighting::LightContext::Create();

code/addons/staticui/ultralight/ultralightrenderer.cc

@@ -180,7 +180,7 @@ UltralightRenderer::UpdateTexture(uint32_t texture_id, ultralight::Ref<ultraligh
     bitmap->UnlockPixels();
 
     // Lock setup command buffer for update
-    CoreGraphics::CmdBufferId setupCmd = CoreGraphics::LockGraphicsSetupCommandBuffer();
+    CoreGraphics::CmdBufferId setupCmd = CoreGraphics::LockGraphicsSetupCommandBuffer("Ultralight texture update");
 
     CoreGraphics::BufferCopy from;
     from.offset = 0;
@@ -215,7 +215,7 @@ UltralightRenderer::UpdateTexture(uint32_t texture_id, ultralight::Ref<ultraligh
                         });
 
     // Finish command buffer
-    CoreGraphics::UnlockGraphicsSetupCommandBuffer();
+    CoreGraphics::UnlockGraphicsSetupCommandBuffer(setupCmd);
 
     // Delete temporary buffer
     CoreGraphics::DestroyBuffer(tempBuf);
@@ -402,7 +402,7 @@ UltralightRenderer::UpdateGeometry(uint32_t geometry_id, const ultralight::Verte
     CoreGraphics::BufferId ibo = CoreGraphics::CreateBuffer(iboInfo);
 
     // Lock setup command buffer for update
-    CoreGraphics::CmdBufferId setupCmd = CoreGraphics::LockGraphicsSetupCommandBuffer();
+    CoreGraphics::CmdBufferId setupCmd = CoreGraphics::LockGraphicsSetupCommandBuffer("Ultralight geometry update");
 
     CoreGraphics::BufferCopy from, to;
     from.offset = 0;
@@ -413,7 +413,7 @@ UltralightRenderer::UpdateGeometry(uint32_t geometry_id, const ultralight::Verte
     CoreGraphics::CmdCopy(setupCmd, ibo, { from }, geo.ibo, { to }, indices.size);
 
     // Finish command buffer
-    CoreGraphics::UnlockGraphicsSetupCommandBuffer();
+    CoreGraphics::UnlockGraphicsSetupCommandBuffer(setupCmd);
 
     // Delete temporary buffer
     CoreGraphics::DestroyBuffer(vbo);

code/foundation/core/config.h

@@ -101,6 +101,10 @@ static const int JobMaxSliceSize = 0xFFFF;
     #define NEBULA_ENABLE_PROFILING (0)
 #endif
 
+#if NEBULA_ENABLE_PROFILING
+#define NEBULA_ENABLE_PERFORMANCE_WARNINGS (1)
+#endif
+
 // max length of a path name
 #define NEBULA_MAXPATH (512)
 

code/foundation/core/coreserver.cc

@@ -5,6 +5,7 @@
 //------------------------------------------------------------------------------
 
 #include "core/coreserver.h"
+#include "threading/thread.h"
 
 namespace Core
 {
@@ -23,6 +24,7 @@ CoreServer::CoreServer() :
     toolDirectory("root:"),
     isOpen(false)
 {
+    Threading::MainThreadId = Threading::Thread::GetMyThreadId();
     __ConstructSingleton;
 }
 

code/foundation/memory/rangeallocator.h

@@ -26,6 +26,7 @@ uint BinFromSize(uint size, uint bucket);
 struct RangeAllocation
 {
     uint offset;
+    uint size;
     uint node;
 
     static constexpr uint OOM = 0xFFFFFFFF;
@@ -186,7 +187,7 @@ RangeAllocator::Alloc(uint size, uint alignment)
     uint alignedSize = size + alignment - 1;
     if (this->freeStorage < alignedSize)
     {
-        return { RangeAllocation::OOM, RangeAllocatorNode::END };
+        return RangeAllocation{ .offset = RangeAllocation::OOM, .size = 0, .node = RangeAllocatorNode::END };
     }
 
     BinIndex minIndex = IndexFromSize(alignedSize, true);
@@ -206,7 +207,7 @@ RangeAllocator::Alloc(uint size, uint alignment)
         // If this means we get 32, it means we're out of buckets that fit
         if (bucket == 0xFFFFFFFF)
         {
-            return { RangeAllocation::OOM, RangeAllocatorNode::END };
+            return RangeAllocation{ .offset = RangeAllocation::OOM, .size = 0, .node = RangeAllocatorNode::END };
         }
 
         // Find any bit, since this bucket has to fit
@@ -269,7 +270,7 @@ RangeAllocator::Alloc(uint size, uint alignment)
         }
     }
 
-    return { node.offset, nodeIndex };
+    return RangeAllocation{ .offset = node.offset, .size = alignedSize, .node = nodeIndex };
 }
 
 //------------------------------------------------------------------------------

code/foundation/threading/safequeue.h

@@ -214,7 +214,7 @@ template<class TYPE> void
 SafeQueue<TYPE>::DequeueAll(Util::Array<TYPE>& outArray)
 {
     this->criticalSection.Enter();
-#if NEBULA_BOUNDSCHECKS
+#if NEBULA_ENABLE_PERFORMANCE_WARNINGS
     n_warn_fmt(outArray.Capacity() >= this->queue.Size(), "SafeQueue::DequeueAll(): (PERFORMANCE) Output array is too small (%d), requires (%d), array will have to grow.\n", outArray.Capacity(), this->queue.Size());
 #endif
     outArray.Clear();
@@ -295,4 +295,3 @@ SafeQueue<TYPE>::EraseMatchingElements(const TYPE& e)
 } // namespace Threading
 //------------------------------------------------------------------------------
 
-

code/foundation/threading/thread.cc

@@ -17,4 +17,6 @@ __ImplementClass(Threading::Thread, 'TRED', Linux::LinuxThread);
 #else
 #error "Thread class not implemented on this platform!"
 #endif
+
+Threading::ThreadId MainThreadId = Threading::InvalidThreadId;
 }

code/foundation/threading/thread.h

@@ -40,4 +40,8 @@ class Thread : public Posix::PosixThread
 #else
 #error "Threading::Thread not implemented on this platform!"
 #endif
+namespace Threading
+{
+extern Threading::ThreadId MainThreadId;
+}
 //------------------------------------------------------------------------------

code/foundation/util/arrayallocatorsafe.h

@@ -142,7 +142,7 @@ class ArrayAllocatorSafe
 
     uint32_t size;
     std::tuple<Util::PinnedArray<MAX_ALLOCS, TYPES>...> objects;
-    Util::Array<Threading::ThreadId> owners;
+    Util::PinnedArray<MAX_ALLOCS, Threading::ThreadId> owners;
 
     Threading::Spinlock allocationLock;
 };

code/foundation/util/bit.h

@@ -242,12 +242,12 @@ Lsb(uint value, byte bit)
 /**
 */
 inline uint 
-BitmaskConvert(uint mask, const uint* table)
+BitmaskConvert(uint mask, const uint* table, const uint numEntries = 0xFFFFFFFF)
 {
     uint ret = 0x0;
     uint usageFlags = mask;
     uint flagIndex = 0;
-    while (usageFlags != 0x0)
+    while (usageFlags != 0x0 && flagIndex < numEntries)
     {
         if (usageFlags & (1 << flagIndex))
         {

code/foundation/util/pinnedarray.h

@@ -717,7 +717,13 @@ PinnedArray<MAX_ALLOCS, TYPE>::GrowTo(SizeT newCapacity)
 
     // Rounded up to the page size so we don't waste memory we allocate anyways
     SizeT totalBytesNeeded = Math::align(totalByteSize, pageSize);
-    n_assert(totalBytesNeeded <= MAX_ALLOCS * sizeof(TYPE));
+
+#if NEBULA_DEBUG
+    if (totalBytesNeeded > MAX_ALLOCS * sizeof(TYPE))
+    {
+        n_printf("[PinnedArray] MAX_ALLOCS '%d' and item size '%d' will waste '%d' byte(s) due to alignment of page size '%d'", MAX_ALLOCS, sizeof(TYPE), totalBytesNeeded - MAX_ALLOCS * sizeof(TYPE), pageSize);
+    }
+#endif
     SizeT roundedUpNewCapacity = totalBytesNeeded / sizeof(TYPE);
     SizeT offset = Math::align(this->capacity * sizeof(TYPE), pageSize);
     if (totalBytesNeeded > offset)

code/physics/physics/streamactorpool.cc

@@ -435,10 +435,11 @@ AddHeightField(PhysicsResource::HeightFieldColliderT* colliderNode, Math::mat4 c
 //------------------------------------------------------------------------------
 /**
 */
-Resources::ResourceUnknownId
-StreamActorPool::InitializeResource(const Ids::Id32 entry, const Util::StringAtom & tag, const Ptr<IO::Stream>& stream, bool immediate)
+Resources::ResourceLoader::ResourceInitOutput
+StreamActorPool::InitializeResource(const ResourceLoadJob& job, const Ptr<IO::Stream>& stream)
 {
-    n_assert(stream.isvalid());    
+    n_assert(stream.isvalid());
+    Resources::ResourceLoader::ResourceInitOutput ret;
 
     /// during the load-phase, we can safetly get the structs
     ActorInfo actorInfo;
@@ -463,21 +464,21 @@ StreamActorPool::InitializeResource(const Ids::Id32 entry, const Util::StringAto
 
                 float radius = collider->data.AsSphereCollider()->radius;
                 physx::PxGeometryHolder geometry = PxSphereGeometry(radius);
-                AddCollider(geometry, material, shape->transform, "Sphere", collider->name, actorInfo, tag, entry);
+                AddCollider(geometry, material, shape->transform, "Sphere", collider->name, actorInfo, job.tag, job.id.loaderInstanceId);
             }
             break;
             case ColliderType_Cube:
             {
                 Math::vector extents = collider->data.AsBoxCollider()->extents;
                 physx::PxGeometryHolder geometry = PxBoxGeometry(Neb2PxVec(extents));
-                AddCollider(geometry, material, shape->transform, "Cube", collider->name, actorInfo, tag, entry);
+                AddCollider(geometry, material, shape->transform, "Cube", collider->name, actorInfo, job.tag, job.id.loaderInstanceId);
             }
             break;
             case ColliderType_Plane:
             {
                 // plane is defined via transform of the actor
                 physx::PxGeometryHolder geometry = PxPlaneGeometry();
-                AddCollider(geometry, material, shape->transform, "Plane", collider->name, actorInfo, tag, entry);
+                AddCollider(geometry, material, shape->transform, "Plane", collider->name, actorInfo, job.tag, job.id.loaderInstanceId);
             }
             break;
             case ColliderType_Capsule:
@@ -486,12 +487,12 @@ StreamActorPool::InitializeResource(const Ids::Id32 entry, const Util::StringAto
                 float radius = capsule->radius;
                 float halfHeight = capsule->halfheight;
                 physx::PxGeometryHolder geometry = PxCapsuleGeometry(radius, halfHeight);
-                AddCollider(geometry, material, shape->transform, "Capsule", collider->name, actorInfo, tag, entry);
+                AddCollider(geometry, material, shape->transform, "Capsule", collider->name, actorInfo, job.tag, job.id.loaderInstanceId);
             }
             break;
             case ColliderType_Mesh:
             {
-                AddMeshColliders(collider->data.AsMeshCollider(), shape->transform, collider->name, material, tag, entry, actorInfo);
+                AddMeshColliders(collider->data.AsMeshCollider(), shape->transform, collider->name, material, job.tag, job.id.loaderInstanceId, actorInfo);
 /*
                 auto mesh = collider->data.AsMeshCollider();
                 MeshTopology type = mesh->type;
@@ -505,7 +506,7 @@ StreamActorPool::InitializeResource(const Ids::Id32 entry, const Util::StringAto
             break;
             case ColliderType_HeightField:
             {
-                AddHeightField(collider->data.AsHeightFieldCollider(), shape->transform, collider->name, material, tag, entry, actorInfo);
+                AddHeightField(collider->data.AsHeightFieldCollider(), shape->transform, collider->name, material, job.tag, job.id.loaderInstanceId, actorInfo);
             }
             break;
             default:
@@ -515,8 +516,7 @@ StreamActorPool::InitializeResource(const Ids::Id32 entry, const Util::StringAto
 
     Ids::Id32 id = allocator.Alloc();
     allocator.Set<Actor_Info>(id, actorInfo);
-
-    ActorResourceId ret = id;
+    ret.id = id;
     return ret;
 }
 

code/physics/physics/streamactorpool.h

@@ -62,7 +62,7 @@ class StreamActorPool : public Resources::ResourceLoader
 
 
     /// perform actual load, override in subclass
-    Resources::ResourceUnknownId InitializeResource(const Ids::Id32 entry, const Util::StringAtom& tag, const Ptr<IO::Stream>& stream, bool immediate) override;
+    ResourceLoader::ResourceInitOutput InitializeResource(const ResourceLoadJob& job, const Ptr<IO::Stream>& stream) override;
     /// unload resource
     void Unload(const Resources::ResourceId id) override;
 

code/render/CMakeLists.txt

@@ -630,7 +630,7 @@ ENDIF()
             )
         fips_dir(terrain/shaders)
             add_shaders(
-                terrain.fx terrain_mesh_generate.fx
+                terrain_include.fxh terrain.fx terrain_tile_write.fx terrain_mesh_generate.fx
             )
 
         fips_dir(vegetation)

code/render/characters/skeletonloader.cc

@@ -16,10 +16,11 @@ __ImplementClass(Characters::SkeletonLoader, 'SSKP', Resources::ResourceLoader)
 //------------------------------------------------------------------------------
 /**
 */
-Resources::ResourceUnknownId
-SkeletonLoader::InitializeResource(const Ids::Id32 entry, const Util::StringAtom& tag, const Ptr<IO::Stream>& stream, bool immediate)
+Resources::ResourceLoader::ResourceInitOutput
+SkeletonLoader::InitializeResource(const ResourceLoadJob& job, const Ptr<IO::Stream>& stream)
 {
     // map buffer
+    Resources::ResourceLoader::ResourceInitOutput ret;
     byte* ptr = (byte*)stream->Map();
 
     // read header
@@ -30,7 +31,7 @@ SkeletonLoader::InitializeResource(const Ids::Id32 entry, const Util::StringAtom
     if (Util::FourCC(header->magic) != NEBULA_NSK3_MAGICNUMBER)
     {
         n_error("StreamSkeletonCache::InitializeResource(): '%s' has invalid file format (magic number doesn't match)!", stream->GetURI().AsString().AsCharPtr());
-        return Resources::InvalidResourceUnknownId;
+        return ret;
     }
 
     Util::FixedArray<SkeletonId> skeletons(header->numSkeletons);
@@ -92,7 +93,7 @@ SkeletonLoader::InitializeResource(const Ids::Id32 entry, const Util::StringAtom
 
     auto id = skeletonResourceAllocator.Alloc();
     skeletonResourceAllocator.Set<0>(id, skeletons);
-    SkeletonResourceId ret = id;
+    ret.id = id;
     return ret;
 }
 

code/render/characters/skeletonloader.h

@@ -20,7 +20,7 @@ class SkeletonLoader : public Resources::ResourceLoader
 private:
 
     /// load character definition from stream
-    Resources::ResourceUnknownId InitializeResource(const Ids::Id32 entry, const Util::StringAtom& tag, const Ptr<IO::Stream>& stream, bool immediate = false) override;
+    ResourceLoader::ResourceInitOutput InitializeResource(const ResourceLoadJob& job, const Ptr<IO::Stream>& stream) override;
     /// unload resource
     void Unload(const Resources::ResourceId id) override;
 

code/render/coreanimation/animationloader.cc

@@ -21,10 +21,11 @@ using namespace Math;
 //------------------------------------------------------------------------------
 /**
 */
-Resources::ResourceUnknownId
-AnimationLoader::InitializeResource(const Ids::Id32 entry, const Util::StringAtom& tag, const Ptr<IO::Stream>& stream, bool immediate)
+Resources::ResourceLoader::ResourceInitOutput
+AnimationLoader::InitializeResource(const ResourceLoadJob& job, const Ptr<IO::Stream>& stream)
 {
     Ptr<AnimKeyBuffer> keyBuffer = nullptr;
+    Resources::ResourceLoader::ResourceInitOutput ret;
 
     // map buffer
     uchar* ptr = (uchar*)stream->Map();
@@ -37,7 +38,7 @@ AnimationLoader::InitializeResource(const Ids::Id32 entry, const Util::StringAto
     if (FourCC(naxHeader->magic) != NEBULA_NAX3_MAGICNUMBER)
     {
         n_error("StreamAnimationLoader::InitializeResource(): '%s' has invalid file format (magic number doesn't match)!", stream->GetURI().AsString().AsCharPtr());
-        return Resources::InvalidResourceUnknownId;
+        return ret;
     }
 
     // load animation if it has clips in it
@@ -134,7 +135,7 @@ AnimationLoader::InitializeResource(const Ids::Id32 entry, const Util::StringAto
 
     auto id = animationResourceAllocator.Alloc();
     animationResourceAllocator.Set<0>(id, animations);
-    AnimationResourceId ret = id;
+    ret.id = id;
     return ret;
 }
 

code/render/coreanimation/animationloader.h

@@ -28,7 +28,7 @@ class AnimationLoader : public Resources::ResourceLoader
     friend class AnimSampleBuffer;
 
     /// perform actual load, override in subclass
-    Resources::ResourceUnknownId InitializeResource(const Ids::Id32 entry, const Util::StringAtom& tag, const Ptr<IO::Stream>& stream, bool immediate = false) override;
+    ResourceLoader::ResourceInitOutput InitializeResource(const ResourceLoadJob& job, const Ptr<IO::Stream>& stream) override;
     /// unload resource
     void Unload(const Resources::ResourceId id) override;
 };