Multiview support for GS (#17508)

Author: CedricGuillemet

packages/dev/core/src/Materials/GaussianSplatting/gaussianSplattingMaterial.ts

@@ -4,7 +4,7 @@ import type { Mesh } from "../../Meshes/mesh";
 import type { Effect, IEffectCreationOptions } from "../../Materials/effect";
 import type { Scene } from "../../scene";
 import type { Matrix } from "../../Maths/math.vector";
-import type { GaussianSplattingMesh } from "../../Meshes";
+import type { GaussianSplattingMesh } from "../../Meshes/GaussianSplatting/gaussianSplattingMesh";
 import { SerializationHelper } from "../../Misc/decorators.serialization";
 import { VertexBuffer } from "../../Buffers/buffer";
 import { MaterialDefines } from "../../Materials/materialDefines";
@@ -150,6 +150,7 @@ export class GaussianSplattingMaterial extends PushMaterial {
         "kernelSize",
         "viewDirectionFactor",
     ];
+    private _sourceMesh: GaussianSplattingMesh | null = null;
     /**
      * Checks whether the material is ready to be rendered for a given mesh.
      * @param mesh The mesh to render
@@ -182,8 +183,12 @@ export class GaussianSplattingMaterial extends PushMaterial {
             return true;
         }
 
+        if (!this._sourceMesh) {
+            return false;
+        }
+
         const engine = scene.getEngine();
-        const gsMesh = mesh as GaussianSplattingMesh;
+        const gsMesh = this._sourceMesh;
 
         // Misc.
         PrepareDefinesForMisc(
@@ -270,6 +275,13 @@ export class GaussianSplattingMaterial extends PushMaterial {
         return true;
     }
 
+    /**
+     * GaussianSplattingMaterial belongs to a single mesh
+     * @param mesh mesh this material belongs to
+     */
+    public setSourceMesh(mesh: GaussianSplattingMesh) {
+        this._sourceMesh = mesh;
+    }
     /**
      * Bind material effect for a specific Gaussian Splatting mesh
      * @param mesh Gaussian splatting mesh
@@ -280,11 +292,16 @@ export class GaussianSplattingMaterial extends PushMaterial {
         const engine = scene.getEngine();
         const camera = scene.activeCamera;
 
-        const renderWidth = engine.getRenderWidth();
-        const renderHeight = engine.getRenderHeight();
+        const renderWidth = engine.getRenderWidth() * camera!.viewport.width;
+        const renderHeight = engine.getRenderHeight() * camera!.viewport.height;
+
+        const gsMaterial = mesh.material as GaussianSplattingMaterial;
+
+        if (!gsMaterial._sourceMesh) {
+            return;
+        }
 
-        const gsMesh = mesh as GaussianSplattingMesh;
-        const gsMaterial = gsMesh.material as GaussianSplattingMaterial;
+        const gsMesh = gsMaterial._sourceMesh;
 
         // check if rigcamera, get number of rigs
         const numberOfRigs = camera?.rigParent?.rigCameras.length || 1;

packages/dev/core/src/Materials/uniformBuffer.ts

@@ -26,6 +26,7 @@ export class UniformBuffer {
     private _buffer: Nullable<DataBuffer>;
     private _buffers: Array<[DataBuffer, Float32Array | undefined]>;
     private _bufferIndex: number;
+    private _bufferUpdatedLastFrame: boolean;
     private _createBufferOnWrite: boolean;
     private _data: number[];
     private _bufferData: Float32Array;
@@ -258,6 +259,7 @@ export class UniformBuffer {
         if ((trackUBOsInFrame === undefined && this._engine._features.trackUbosInFrame) || trackUBOsInFrame === true) {
             this._buffers = [];
             this._bufferIndex = -1;
+            this._bufferUpdatedLastFrame = false;
             this._createBufferOnWrite = false;
             this._currentFrameId = 0;
             this._trackUBOsInFrame = true;
@@ -691,6 +693,8 @@ export class UniformBuffer {
             }
         }
 
+        this._bufferUpdatedLastFrame = true;
+
         this._engine.updateUniformBuffer(this._buffer, this._bufferData);
 
         if (this._engine._features._collectUbosUpdatedInFrame) {
@@ -720,7 +724,11 @@ export class UniformBuffer {
             this._currentFrameId = this._engine.frameId;
             this._createBufferOnWrite = false;
             if (this._buffers && this._buffers.length > 0) {
-                this._needSync = this._bufferIndex !== 0;
+                if (this._buffers.length === 1) {
+                    this._needSync = !this._bufferUpdatedLastFrame;
+                } else {
+                    this._needSync = this._bufferIndex !== 0;
+                }
                 this._bufferIndex = 0;
                 this._buffer = this._buffers[this._bufferIndex][0];
             } else {