From ea1c0cee6eb815845bb69febdf5effa397de27e6 Mon Sep 17 00:00:00 2001 From: Cedric Guillemet <1312968+CedricGuillemet@users.noreply.github.com> Date: Tue, 23 Sep 2025 18:03:00 +0200 Subject: [PATCH 1/7] Native GS Experiment --- .../gaussianSplattingMesh.ts | 112 ++++++++++-------- 1 file changed, 64 insertions(+), 48 deletions(-) diff --git a/packages/dev/core/src/Meshes/GaussianSplatting/gaussianSplattingMesh.ts b/packages/dev/core/src/Meshes/GaussianSplatting/gaussianSplattingMesh.ts index c9954e753ac..1b9a13a2090 100644 --- a/packages/dev/core/src/Meshes/GaussianSplatting/gaussianSplattingMesh.ts +++ b/packages/dev/core/src/Meshes/GaussianSplatting/gaussianSplattingMesh.ts @@ -19,6 +19,10 @@ import type { Material } from "core/Materials/material"; import { Scalar } from "core/Maths/math.scalar"; import { runCoroutineSync, runCoroutineAsync, createYieldingScheduler, type Coroutine } from "core/Misc/coroutine"; import { EngineStore } from "core/Engines/engineStore"; +import type { INative } from "core/Engines/Native/nativeInterfaces"; + +// eslint-disable-next-line @typescript-eslint/naming-convention +declare const _native: INative; interface IDelayedTextureUpdate { covA: Uint16Array; @@ -445,7 +449,7 @@ export class GaussianSplattingMesh extends Mesh { return false; } - if (!this._readyToDisplay) { + if (!_native && !this._readyToDisplay) { // mesh is ready when worker has done at least 1 sorting this._postToWorker(true); return false; @@ -468,10 +472,15 @@ export class GaussianSplattingMesh extends Mesh { if (forced || Math.abs(dot - 1) >= 0.01) { this._oldDirection.copyFrom(TmpVectors.Vector3[2]); this._frameIdLastUpdate = frameId; - this._canPostToWorker = false; - this._worker.postMessage({ view: this._modelViewMatrix.m, depthMix: this._depthMix, useRightHandedSystem: this._scene.useRightHandedSystem }, [ - this._depthMix.buffer, - ]); + if (_native) { + // @ts-expect-error: sortGS is a native function not recognized by TypeScript + sortGS(this._modelViewMatrix, this._splatPositions, this._splatIndex, this._scene.useRightHandedSystem); + } else { + this._canPostToWorker = false; + this._worker.postMessage({ view: this._modelViewMatrix.m, depthMix: this._depthMix, useRightHandedSystem: this._scene.useRightHandedSystem }, [ + this._depthMix.buffer, + ]); + } } } } @@ -1386,7 +1395,9 @@ export class GaussianSplattingMesh extends Mesh { this._delayedTextureUpdate = { covA: covA, covB: covB, colors: colorArray, centers: this._splatPositions!, sh: sh }; const positions = Float32Array.from(this._splatPositions!); const vertexCount = this._vertexCount; - this._worker!.postMessage({ positions, vertexCount }, [positions.buffer]); + if (this._worker) { + this._worker.postMessage({ positions, vertexCount }, [positions.buffer]); + } this._postToWorker(true); } else { @@ -1474,7 +1485,9 @@ export class GaussianSplattingMesh extends Mesh { // sort will be dirty here as just finished filled positions will not be sorted const positions = Float32Array.from(this._splatPositions); const vertexCount = this._vertexCount; - this._worker!.postMessage({ positions, vertexCount }, [positions.buffer]); + if (this._worker) { + this._worker.postMessage({ positions, vertexCount }, [positions.buffer]); + } this._sortIsDirty = true; } else { for (let i = 0; i < vertexCount; i++) { @@ -1564,51 +1577,54 @@ export class GaussianSplattingMesh extends Mesh { // Start the worker thread this._worker?.terminate(); - this._worker = new Worker( - URL.createObjectURL( - new Blob(["(", GaussianSplattingMesh._CreateWorker.toString(), ")(self)"], { - type: "application/javascript", - }) - ) - ); - this._depthMix = new BigInt64Array(this._vertexCount); - const positions = Float32Array.from(this._splatPositions!); - const vertexCount = this._vertexCount; + if (!_native) { + this._worker = new Worker( + URL.createObjectURL( + new Blob(["(", GaussianSplattingMesh._CreateWorker.toString(), ")(self)"], { + type: "application/javascript", + }) + ) + ); + + this._depthMix = new BigInt64Array(this._vertexCount); + const positions = Float32Array.from(this._splatPositions!); + const vertexCount = this._vertexCount; - this._worker.postMessage({ positions, vertexCount }, [positions.buffer]); + this._worker.postMessage({ positions, vertexCount }, [positions.buffer]); - this._worker.onmessage = (e) => { - this._depthMix = e.data.depthMix; - const indexMix = new Uint32Array(e.data.depthMix.buffer); - if (this._splatIndex) { - for (let j = 0; j < this._vertexCount; j++) { - this._splatIndex[j] = indexMix[2 * j]; + this._worker.onmessage = (e) => { + this._depthMix = e.data.depthMix; + const indexMix = new Uint32Array(e.data.depthMix.buffer); + if (this._splatIndex) { + for (let j = 0; j < this._vertexCount; j++) { + this._splatIndex[j] = indexMix[2 * j]; + } } - } - if (this._delayedTextureUpdate) { - const textureSize = this._getTextureSize(vertexCount); - this._updateSubTextures( - this._delayedTextureUpdate.centers, - this._delayedTextureUpdate.covA, - this._delayedTextureUpdate.covB, - this._delayedTextureUpdate.colors, - 0, - textureSize.y, - this._delayedTextureUpdate.sh - ); - this._delayedTextureUpdate = null; - } - this.thinInstanceBufferUpdated("splatIndex"); - this._canPostToWorker = true; - this._readyToDisplay = true; - // sort is dirty when GS is visible for progressive update with a this message arriving but positions were partially filled - // another update needs to be kicked. The kick can't happen just when the position buffer is ready because _canPostToWorker might be false. - if (this._sortIsDirty) { - this._postToWorker(true); - this._sortIsDirty = false; - } - }; + if (this._delayedTextureUpdate) { + const textureSize = this._getTextureSize(vertexCount); + this._updateSubTextures( + this._delayedTextureUpdate.centers, + this._delayedTextureUpdate.covA, + this._delayedTextureUpdate.covB, + this._delayedTextureUpdate.colors, + 0, + textureSize.y, + this._delayedTextureUpdate.sh + ); + this._delayedTextureUpdate = null; + } + this.thinInstanceBufferUpdated("splatIndex"); + this._canPostToWorker = true; + this._readyToDisplay = true; + // sort is dirty when GS is visible for progressive update with a this message arriving but positions were partially filled + // another update needs to be kicked. The kick can't happen just when the position buffer is ready because _canPostToWorker might be false. + if (this._sortIsDirty) { + this._postToWorker(true); + this._sortIsDirty = false; + } + }; + } } private _getTextureSize(length: number): Vector2 { From fea5e3dcf3b4e9a5864004ae7f8d4088fc4611ba Mon Sep 17 00:00:00 2001 From: Cedric Guillemet <1312968+CedricGuillemet@users.noreply.github.com> Date: Wed, 24 Sep 2025 11:29:48 +0200 Subject: [PATCH 2/7] pixels on screen! --- .../GaussianSplatting/gaussianSplattingMesh.ts | 13 ++++++++----- .../core/src/Shaders/gaussianSplatting.vertex.fx | 4 ++-- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/packages/dev/core/src/Meshes/GaussianSplatting/gaussianSplattingMesh.ts b/packages/dev/core/src/Meshes/GaussianSplatting/gaussianSplattingMesh.ts index 1b9a13a2090..8cc76e8b4a6 100644 --- a/packages/dev/core/src/Meshes/GaussianSplatting/gaussianSplattingMesh.ts +++ b/packages/dev/core/src/Meshes/GaussianSplatting/gaussianSplattingMesh.ts @@ -460,7 +460,7 @@ export class GaussianSplattingMesh extends Mesh { /** @internal */ public _postToWorker(forced = false): void { const frameId = this.getScene().getFrameId(); - if ((forced || frameId !== this._frameIdLastUpdate) && this._worker && this._scene.activeCamera && this._canPostToWorker) { + if ((forced || frameId !== this._frameIdLastUpdate) && ((this._worker && this._canPostToWorker) || _native) && this._scene.activeCamera) { const cameraMatrix = this._scene.activeCamera.getViewMatrix(); this.getWorldMatrix().multiplyToRef(cameraMatrix, this._modelViewMatrix); cameraMatrix.invertToRef(TmpVectors.Matrix[0]); @@ -475,9 +475,12 @@ export class GaussianSplattingMesh extends Mesh { if (_native) { // @ts-expect-error: sortGS is a native function not recognized by TypeScript sortGS(this._modelViewMatrix, this._splatPositions, this._splatIndex, this._scene.useRightHandedSystem); + this.thinInstanceBufferUpdated("splatIndex"); + this._canPostToWorker = true; + this._readyToDisplay = true; } else { this._canPostToWorker = false; - this._worker.postMessage({ view: this._modelViewMatrix.m, depthMix: this._depthMix, useRightHandedSystem: this._scene.useRightHandedSystem }, [ + this._worker!.postMessage({ view: this._modelViewMatrix.m, depthMix: this._depthMix, useRightHandedSystem: this._scene.useRightHandedSystem }, [ this._depthMix.buffer, ]); } @@ -1537,9 +1540,9 @@ export class GaussianSplattingMesh extends Mesh { // in case size is different private _updateSplatIndexBuffer(vertexCount: number): void { if (!this._splatIndex || vertexCount > this._splatIndex.length) { - this._splatIndex = new Float32Array(vertexCount); + this._splatIndex = new Float32Array(vertexCount * 4); - this.thinInstanceSetBuffer("splatIndex", this._splatIndex, 1, false); + this.thinInstanceSetBuffer("splatIndex", this._splatIndex, 4, false); } this.forcedInstanceCount = vertexCount; } @@ -1598,7 +1601,7 @@ export class GaussianSplattingMesh extends Mesh { const indexMix = new Uint32Array(e.data.depthMix.buffer); if (this._splatIndex) { for (let j = 0; j < this._vertexCount; j++) { - this._splatIndex[j] = indexMix[2 * j]; + this._splatIndex[j * 4] = indexMix[2 * j]; } } if (this._delayedTextureUpdate) { diff --git a/packages/dev/core/src/Shaders/gaussianSplatting.vertex.fx b/packages/dev/core/src/Shaders/gaussianSplatting.vertex.fx index 57f78b0ab70..556a03ed175 100644 --- a/packages/dev/core/src/Shaders/gaussianSplatting.vertex.fx +++ b/packages/dev/core/src/Shaders/gaussianSplatting.vertex.fx @@ -11,7 +11,7 @@ #include // Attributes -attribute float splatIndex; +attribute vec4 splatIndex; // Uniforms uniform vec2 invViewport; @@ -43,7 +43,7 @@ varying vec2 vPosition; #include void main () { - Splat splat = readSplat(splatIndex); + Splat splat = readSplat(splatIndex.x); vec3 covA = splat.covA.xyz; vec3 covB = vec3(splat.covA.w, splat.covB.xy); From ae9271bcb379479ff54b9103774b8e756340cc5d Mon Sep 17 00:00:00 2001 From: Cedric Guillemet <1312968+CedricGuillemet@users.noreply.github.com> Date: Wed, 26 Nov 2025 11:55:05 +0100 Subject: [PATCH 3/7] texture format --- packages/dev/core/src/Engines/Native/nativeHelpers.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/dev/core/src/Engines/Native/nativeHelpers.ts b/packages/dev/core/src/Engines/Native/nativeHelpers.ts index b06218af344..5c3e8f202d4 100644 --- a/packages/dev/core/src/Engines/Native/nativeHelpers.ts +++ b/packages/dev/core/src/Engines/Native/nativeHelpers.ts @@ -53,6 +53,13 @@ export function getNativeTextureFormat(format: number, type: number): number { } break; } + case Constants.TEXTUREFORMAT_RGBA_INTEGER: { + switch (type) { + case Constants.TEXTURETYPE_UNSIGNED_INTEGER: + return _native.Engine.TEXTURE_FORMAT_RGBA32U; + } + break; + } case Constants.TEXTUREFORMAT_RGBA: { switch (type) { case Constants.TEXTURETYPE_UNSIGNED_BYTE: From e1bc9e9e4532a99f12e8fbc3646acea9521722fe Mon Sep 17 00:00:00 2001 From: Cedric Guillemet <1312968+CedricGuillemet@users.noreply.github.com> Date: Wed, 26 Nov 2025 15:17:45 +0100 Subject: [PATCH 4/7] Clean up --- .vscode/launch.json | 2 +- .../src/Engines/Native/nativeInterfaces.ts | 4 ++ .../gaussianSplattingMesh.ts | 52 ++++++++++++++----- 3 files changed, 44 insertions(+), 14 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 68f6fd18489..4c8d6b143dc 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -118,7 +118,7 @@ "type": "chrome", "request": "launch", "name": "Launch Playground (Chrome)", - "url": "http://localhost:1338/#CID4NN#203", + "url": "http://localhost:1338", "preLaunchTask": "Playground Serve for core (Dev)" }, { diff --git a/packages/dev/core/src/Engines/Native/nativeInterfaces.ts b/packages/dev/core/src/Engines/Native/nativeInterfaces.ts index ddf484c5055..92f28e0b368 100644 --- a/packages/dev/core/src/Engines/Native/nativeInterfaces.ts +++ b/packages/dev/core/src/Engines/Native/nativeInterfaces.ts @@ -2,6 +2,7 @@ import type { DeviceType } from "../../DeviceInput/InputDevices/deviceEnums"; import type { IDeviceInputSystem } from "../../DeviceInput/inputInterfaces"; import type { InternalTexture } from "../../Materials/Textures/internalTexture"; +import type { Matrix } from "../../Maths"; import type { Nullable } from "../../types"; import type { ICanvas, IImage, IPath2D } from "../ICanvas"; import type { NativeData, NativeDataStream } from "./nativeDataStream"; @@ -445,4 +446,7 @@ export interface INative { disablePerformanceLogging?(): void; startPerformanceCounter?(counter: string): unknown; endPerformanceCounter?(counter: unknown): void; + + // GaussianSplatting + sortGS(modelViewMatrix: Matrix, splatPositions: Float32Array, splatIndex: Float32Array, useRightHandedSystem: boolean): void; } diff --git a/packages/dev/core/src/Meshes/GaussianSplatting/gaussianSplattingMesh.ts b/packages/dev/core/src/Meshes/GaussianSplatting/gaussianSplattingMesh.ts index c08ff08448d..71bd912da5e 100644 --- a/packages/dev/core/src/Meshes/GaussianSplatting/gaussianSplattingMesh.ts +++ b/packages/dev/core/src/Meshes/GaussianSplatting/gaussianSplattingMesh.ts @@ -500,7 +500,29 @@ export class GaussianSplattingMesh extends Mesh { /** @internal */ public _postToWorker(forced = false): void { const frameId = this.getScene().getFrameId(); - if ((forced || frameId !== this._frameIdLastUpdate) && ((this._worker && this._canPostToWorker) || _native) && this._scene.activeCamera) { + if (_native && _native.sortGS) { + if ((forced || frameId !== this._frameIdLastUpdate) && this._scene.activeCamera) { + const cameraMatrix = this._scene.activeCamera.getViewMatrix(); + this.getWorldMatrix().multiplyToRef(cameraMatrix, this._modelViewMatrix); + cameraMatrix.invertToRef(TmpVectors.Matrix[0]); + this.getWorldMatrix().multiplyToRef(TmpVectors.Matrix[0], TmpVectors.Matrix[1]); + Vector3.TransformNormalToRef(Vector3.Forward(this._scene.useRightHandedSystem), TmpVectors.Matrix[1], TmpVectors.Vector3[2]); + TmpVectors.Vector3[2].normalize(); + + const dot = Vector3.Dot(TmpVectors.Vector3[2], this._oldDirection); + if (forced || Math.abs(dot - 1) >= 0.01) { + this._oldDirection.copyFrom(TmpVectors.Vector3[2]); + this._frameIdLastUpdate = frameId; + + _native.sortGS(this._modelViewMatrix, this._splatPositions!, this._splatIndex!, this._scene.useRightHandedSystem); + this.thinInstanceBufferUpdated("splatIndex"); + this._canPostToWorker = true; + this._readyToDisplay = true; + } + } + return; + } + if ((forced || frameId !== this._frameIdLastUpdate) && this._worker && this._scene.activeCamera && this._canPostToWorker) { const cameraMatrix = this._scene.activeCamera.getViewMatrix(); this.getWorldMatrix().multiplyToRef(cameraMatrix, this._modelViewMatrix); cameraMatrix.invertToRef(TmpVectors.Matrix[0]); @@ -512,18 +534,10 @@ export class GaussianSplattingMesh extends Mesh { if (forced || Math.abs(dot - 1) >= 0.01) { this._oldDirection.copyFrom(TmpVectors.Vector3[2]); this._frameIdLastUpdate = frameId; - if (_native) { - // @ts-expect-error: sortGS is a native function not recognized by TypeScript - sortGS(this._modelViewMatrix, this._splatPositions, this._splatIndex, this._scene.useRightHandedSystem); - this.thinInstanceBufferUpdated("splatIndex"); - this._canPostToWorker = true; - this._readyToDisplay = true; - } else { - this._canPostToWorker = false; - this._worker!.postMessage({ view: this._modelViewMatrix.m, depthMix: this._depthMix, useRightHandedSystem: this._scene.useRightHandedSystem }, [ - this._depthMix.buffer, - ]); - } + this._canPostToWorker = false; + this._worker.postMessage({ view: this._modelViewMatrix.m, depthMix: this._depthMix, useRightHandedSystem: this._scene.useRightHandedSystem }, [ + this._depthMix.buffer, + ]); } } } @@ -1644,6 +1658,18 @@ export class GaussianSplattingMesh extends Mesh { // Start the worker thread this._worker?.terminate(); + if (!this._worker) { + return; + } + + this._worker = new Worker( + URL.createObjectURL( + new Blob(["(", GaussianSplattingMesh._CreateWorker.toString(), ")(self)"], { + type: "application/javascript", + }) + ) + ); + const vertexCountPadded = (this._vertexCount + 15) & ~0xf; this._depthMix = new BigInt64Array(vertexCountPadded); const positions = Float32Array.from(this._splatPositions!); From 59ae30b91d345d636ef1ef30b3f18048caefcebe Mon Sep 17 00:00:00 2001 From: Cedric Guillemet <1312968+CedricGuillemet@users.noreply.github.com> Date: Wed, 26 Nov 2025 15:19:41 +0100 Subject: [PATCH 5/7] early exit --- .../src/Meshes/GaussianSplatting/gaussianSplattingMesh.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/dev/core/src/Meshes/GaussianSplatting/gaussianSplattingMesh.ts b/packages/dev/core/src/Meshes/GaussianSplatting/gaussianSplattingMesh.ts index 71bd912da5e..72b2cbf0274 100644 --- a/packages/dev/core/src/Meshes/GaussianSplatting/gaussianSplattingMesh.ts +++ b/packages/dev/core/src/Meshes/GaussianSplatting/gaussianSplattingMesh.ts @@ -1655,13 +1655,13 @@ export class GaussianSplattingMesh extends Mesh { } this._updateSplatIndexBuffer(this._vertexCount); - // Start the worker thread - this._worker?.terminate(); - - if (!this._worker) { + if (_native) { return; } + // Start the worker thread + this._worker?.terminate(); + this._worker = new Worker( URL.createObjectURL( new Blob(["(", GaussianSplattingMesh._CreateWorker.toString(), ")(self)"], { From 5f538abc7f82e80190795293acfae3c812bfdc84 Mon Sep 17 00:00:00 2001 From: Cedric Guillemet <1312968+CedricGuillemet@users.noreply.github.com> Date: Wed, 26 Nov 2025 15:42:59 +0100 Subject: [PATCH 6/7] removed empty line --- .../core/src/Meshes/GaussianSplatting/gaussianSplattingMesh.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/dev/core/src/Meshes/GaussianSplatting/gaussianSplattingMesh.ts b/packages/dev/core/src/Meshes/GaussianSplatting/gaussianSplattingMesh.ts index 72b2cbf0274..698ca4c188d 100644 --- a/packages/dev/core/src/Meshes/GaussianSplatting/gaussianSplattingMesh.ts +++ b/packages/dev/core/src/Meshes/GaussianSplatting/gaussianSplattingMesh.ts @@ -1661,7 +1661,6 @@ export class GaussianSplattingMesh extends Mesh { // Start the worker thread this._worker?.terminate(); - this._worker = new Worker( URL.createObjectURL( new Blob(["(", GaussianSplattingMesh._CreateWorker.toString(), ")(self)"], { From 9c30c27f3dec19f542dcf1a5bbb7838ddc3fc0aa Mon Sep 17 00:00:00 2001 From: Cedric Guillemet <1312968+CedricGuillemet@users.noreply.github.com> Date: Wed, 26 Nov 2025 16:09:19 +0100 Subject: [PATCH 7/7] fix import --- packages/dev/core/src/Engines/Native/nativeInterfaces.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/dev/core/src/Engines/Native/nativeInterfaces.ts b/packages/dev/core/src/Engines/Native/nativeInterfaces.ts index 92f28e0b368..d291870afee 100644 --- a/packages/dev/core/src/Engines/Native/nativeInterfaces.ts +++ b/packages/dev/core/src/Engines/Native/nativeInterfaces.ts @@ -2,7 +2,7 @@ import type { DeviceType } from "../../DeviceInput/InputDevices/deviceEnums"; import type { IDeviceInputSystem } from "../../DeviceInput/inputInterfaces"; import type { InternalTexture } from "../../Materials/Textures/internalTexture"; -import type { Matrix } from "../../Maths"; +import type { Matrix } from "../../Maths/math.vector"; import type { Nullable } from "../../types"; import type { ICanvas, IImage, IPath2D } from "../ICanvas"; import type { NativeData, NativeDataStream } from "./nativeDataStream";