Merge branch 'main' into package-updates-october

CHANGES.md

@@ -8,10 +8,12 @@
 
 - Added `enableVerticalExaggeration` option to models. Set this value to `false` to prevent model exaggeration when `Scene.verticalExaggeration` is set to a value other than `1.0`. [#12141](https://github.com/CesiumGS/cesium/pull/12141)
 - Added `CallbackPositionProperty` to allow lazy entity position evaluation. [#12170](https://github.com/CesiumGS/cesium/pull/12170)
+- Added `Scene.prototype.pickMetadata` and `Scene.prototype.pickMetadataSchema`, enabling experimental support for picking property textures or property attributes [#12075](https://github.com/CesiumGS/cesium/pull/12075)
 - Added experimental support for the `NGA_gpm_local` glTF extension, for GPM 1.2 [#12204](https://github.com/CesiumGS/cesium/pull/12204)
 
 ##### Fixes :wrench:
 
+- Fix Texture errors when using a `HTMLVideoElement`. [#12219](https://github.com/CesiumGS/cesium/issues/12219)
 - Use first geometryBuffer if no best match found in I3SNode [#12132](https://github.com/CesiumGS/cesium/pull/12132)
 - Update type definitions to allow undefined for optional parameters [#12193](https://github.com/CesiumGS/cesium/pull/12193)
 - Reverts Firefox OIT temporary fix [#4815] and Firefox test failure fix [#5047]

packages/engine/Source/Renderer/DrawCommand.js

@@ -43,6 +43,8 @@ function DrawCommand(options) {
   this._owner = options.owner;
   this._debugOverlappingFrustums = 0;
   this._pickId = options.pickId;
+  this._pickMetadataAllowed = options.pickMetadataAllowed === true;
+  this._pickedMetadataInfo = undefined;
 
   // Set initial flags.
   this._flags = 0;
@@ -514,7 +516,7 @@ Object.defineProperties(DrawCommand.prototype, {
    * during the pick pass.
    *
    * @memberof DrawCommand.prototype
-   * @type {string}
+   * @type {string|undefined}
    * @default undefined
    */
   pickId: {
@@ -528,6 +530,44 @@ Object.defineProperties(DrawCommand.prototype, {
       }
     },
   },
+
+  /**
+   * Whether metadata picking is allowed.
+   *
+   * This is essentially only set to `true` for draw commands that are
+   * part of a `ModelDrawCommand`, to check whether a derived command
+   * for metadata picking has to be created.
+   *
+   * @memberof DrawCommand.prototype
+   * @type {boolean}
+   * @default undefined
+   * @private
+   */
+  pickMetadataAllowed: {
+    get: function () {
+      return this._pickMetadataAllowed;
+    },
+  },
+
+  /**
+   * Information about picked metadata.
+   *
+   * @memberof DrawCommand.prototype
+   * @type {PickedMetadataInfo|undefined}
+   * @default undefined
+   */
+  pickedMetadataInfo: {
+    get: function () {
+      return this._pickedMetadataInfo;
+    },
+    set: function (value) {
+      if (this._pickedMetadataInfo !== value) {
+        this._pickedMetadataInfo = value;
+        this.dirty = true;
+      }
+    },
+  },
+
   /**
    * Whether this command should be executed in the pick pass only.
    *
@@ -593,6 +633,8 @@ DrawCommand.shallowClone = function (command, result) {
   result._owner = command._owner;
   result._debugOverlappingFrustums = command._debugOverlappingFrustums;
   result._pickId = command._pickId;
+  result._pickMetadataAllowed = command._pickMetadataAllowed;
+  result._pickedMetadataInfo = command._pickedMetadataInfo;
   result._flags = command._flags;
 
   result.dirty = true;

packages/engine/Source/Renderer/Texture.js

@@ -60,11 +60,12 @@ function Texture(options) {
 
   let { width, height } = options;
   if (defined(source)) {
+    // Make sure we are using the element's intrinsic width and height where available
     if (!defined(width)) {
-      width = defaultValue(source.videoWidth, source.width);
+      width = source.videoWidth ?? source.naturalWidth ?? source.width;
     }
     if (!defined(height)) {
-      height = defaultValue(source.videoHeight, source.height);
+      height = source.videoHeight ?? source.naturalHeight ?? source.height;
     }
   }
 
@@ -810,7 +811,17 @@ Texture.prototype.copyFrom = function (options) {
   gl.activeTexture(gl.TEXTURE0);
   gl.bindTexture(target, this._texture);
 
-  const { width, height, arrayBufferView } = source;
+  let { width, height } = source;
+  const arrayBufferView = source.arrayBufferView;
+
+  // Make sure we are using the element's intrinsic width and height where available
+  if (defined(source.videoWidth) && defined(source.videoHeight)) {
+    width = source.videoWidth;
+    height = source.videoHeight;
+  } else if (defined(source.naturalWidth) && defined(source.naturalHeight)) {
+    width = source.naturalWidth;
+    height = source.naturalHeight;
+  }
 
   const textureWidth = this._width;
   const textureHeight = this._height;

packages/engine/Source/Scene/DerivedCommand.js

@@ -2,6 +2,8 @@ import defined from "../Core/defined.js";
 import DrawCommand from "../Renderer/DrawCommand.js";
 import RenderState from "../Renderer/RenderState.js";
 import ShaderSource from "../Renderer/ShaderSource.js";
+import MetadataType from "./MetadataType.js";
+import MetadataPickingPipelineStage from "./Model/MetadataPickingPipelineStage.js";
 
 /**
  * @private
@@ -343,6 +345,227 @@ DerivedCommand.createPickDerivedCommand = function (
   return result;
 };
 
+/**
+ * Replaces the value of the specified 'define' directive identifier
+ * with the given value.
+ *
+ * The given defines are the parts of the define directives that are
+ * stored in the `ShaderSource`. For example, the defines may be
+ * `["EXAMPLE", "EXAMPLE_VALUE 123"]`
+ *
+ * Calling `replaceDefine(defines, "EXAMPLE", 999)` will result in
+ * the defines being
+ * `["EXAMPLE 999", "EXAMPLE_VALUE 123"]`
+ *
+ * @param {string[]} defines The define directive identifiers
+ * @param {string} defineName The name (identifier) of the define directive
+ * @param {any} newDefineValue The new value whose string representation
+ * will become the token string for the define directive
+ * @private
+ */
+function replaceDefine(defines, defineName, newDefineValue) {
+  const n = defines.length;
+  for (let i = 0; i < n; i++) {
+    const define = defines[i];
+    const tokens = define.trimStart().split(/\s+/);
+    if (tokens[0] === defineName) {
+      defines[i] = `${defineName} ${newDefineValue}`;
+    }
+  }
+}
+
+/**
+ * Returns the component count for the given class property, or
+ * its array length if it is an array.
+ *
+ * This will be
+ * `[1, 2, 3, 4]` for `[SCALAR, VEC2, VEC3, VEC4`] types,
+ * or the array length if it is an array.
+ *
+ * @param {MetadataClassProperty} classProperty The class property
+ * @returns {number} The component count
+ * @private
+ */
+function getComponentCount(classProperty) {
+  if (!classProperty.isArray) {
+    return MetadataType.getComponentCount(classProperty.type);
+  }
+  return classProperty.arrayLength;
+}
+
+/**
+ * Returns the type that the given class property has in a GLSL shader.
+ *
+ * It returns the same string as `PropertyTextureProperty.prototype.getGlslType`
+ * for a property texture property with the given class property
+ *
+ * @param {MetadataClassProperty} classProperty The class property
+ * @returns {string} The GLSL shader type string for the property
+ */
+function getGlslType(classProperty) {
+  const componentCount = getComponentCount(classProperty);
+  if (classProperty.normalized) {
+    if (componentCount === 1) {
+      return "float";
+    }
+    return `vec${componentCount}`;
+  }
+  if (componentCount === 1) {
+    return "int";
+  }
+  return `ivec${componentCount}`;
+}
+
+/**
+ * Creates a new `ShaderProgram` from the given input that renders metadata
+ * values into the frame buffer, according to the given picked metadata info.
+ *
+ * This will update the `defines` of the fragment shader of the given shader
+ * program, by setting `METADATA_PICKING_ENABLED`, and updating the
+ * `METADATA_PICKING_VALUE_*` defines so that they reflect the components
+ * of the metadata that should be written into the RGBA (vec4) that
+ * ends up as the 'color' in the frame buffer.
+ *
+ * The RGBA values will eventually be converted back into an actual metadata
+ * value in `Picking.js`, by calling `MetadataPicking.decodeMetadataValues`.
+ *
+ * @param {Context} context The context
+ * @param {ShaderProgram} shaderProgram The shader program
+ * @param {PickedMetadataInfo} pickedMetadataInfo The picked metadata info
+ * @returns {ShaderProgram} The new shader program
+ * @private
+ */
+function getPickMetadataShaderProgram(
+  context,
+  shaderProgram,
+  pickedMetadataInfo,
+) {
+  const schemaId = pickedMetadataInfo.schemaId;
+  const className = pickedMetadataInfo.className;
+  const propertyName = pickedMetadataInfo.propertyName;
+  const keyword = `pickMetadata-${schemaId}-${className}-${propertyName}`;
+  const shader = context.shaderCache.getDerivedShaderProgram(
+    shaderProgram,
+    keyword,
+  );
+  if (defined(shader)) {
+    return shader;
+  }
+
+  const classProperty = pickedMetadataInfo.classProperty;
+  const glslType = getGlslType(classProperty);
+
+  // Define the components that will go into the output `metadataValues`.
+  // By default, all of them are 0.0.
+  const sourceValueStrings = ["0.0", "0.0", "0.0", "0.0"];
+  const componentCount = getComponentCount(classProperty);
+  if (componentCount === 1) {
+    // When the property is a scalar, store its value directly
+    // in `metadataValues.x`
+    sourceValueStrings[0] = `float(value)`;
+  } else {
+    // When the property is an array, store the array elements
+    // in `metadataValues.x/y/z/w`
+    const components = ["x", "y", "z", "w"];
+    for (let i = 0; i < componentCount; i++) {
+      const component = components[i];
+      const valueString = `value.${component}`;
+      sourceValueStrings[i] = `float(${valueString})`;
+    }
+  }
+
+  // Make sure that the `metadataValues` components are all in
+  // the range [0, 1] (which will result in RGBA components
+  // in [0, 255] during rendering)
+  if (!classProperty.normalized) {
+    for (let i = 0; i < componentCount; i++) {
+      sourceValueStrings[i] += " / 255.0";
+    }
+  }
+
+  const newDefines = shaderProgram.fragmentShaderSource.defines.slice();
+  newDefines.push(MetadataPickingPipelineStage.METADATA_PICKING_ENABLED);
+
+  // Replace the defines of the shader, using the type, property
+  // access, and value components  that have been determined
+  replaceDefine(
+    newDefines,
+    MetadataPickingPipelineStage.METADATA_PICKING_VALUE_TYPE,
+    glslType,
+  );
+  replaceDefine(
+    newDefines,
+    MetadataPickingPipelineStage.METADATA_PICKING_VALUE_STRING,
+    `metadata.${propertyName}`,
+  );
+  replaceDefine(
+    newDefines,
+    MetadataPickingPipelineStage.METADATA_PICKING_VALUE_COMPONENT_X,
+    sourceValueStrings[0],
+  );
+  replaceDefine(
+    newDefines,
+    MetadataPickingPipelineStage.METADATA_PICKING_VALUE_COMPONENT_Y,
+    sourceValueStrings[1],
+  );
+  replaceDefine(
+    newDefines,
+    MetadataPickingPipelineStage.METADATA_PICKING_VALUE_COMPONENT_Z,
+    sourceValueStrings[2],
+  );
+  replaceDefine(
+    newDefines,
+    MetadataPickingPipelineStage.METADATA_PICKING_VALUE_COMPONENT_W,
+    sourceValueStrings[3],
+  );
+
+  const newFragmentShaderSource = new ShaderSource({
+    sources: shaderProgram.fragmentShaderSource.sources,
+    defines: newDefines,
+  });
+  const newShader = context.shaderCache.createDerivedShaderProgram(
+    shaderProgram,
+    keyword,
+    {
+      vertexShaderSource: shaderProgram.vertexShaderSource,
+      fragmentShaderSource: newFragmentShaderSource,
+      attributeLocations: shaderProgram._attributeLocations,
+    },
+  );
+  return newShader;
+}
+
+/**
+ * @private
+ */
+DerivedCommand.createPickMetadataDerivedCommand = function (
+  scene,
+  command,
+  context,
+  result,
+) {
+  if (!defined(result)) {
+    result = {};
+  }
+  result.pickMetadataCommand = DrawCommand.shallowClone(
+    command,
+    result.pickMetadataCommand,
+  );
+
+  result.pickMetadataCommand.shaderProgram = getPickMetadataShaderProgram(
+    context,
+    command.shaderProgram,
+    command.pickedMetadataInfo,
+  );
+  result.pickMetadataCommand.renderState = getPickRenderState(
+    scene,
+    command.renderState,
+  );
+  result.shaderProgramId = command.shaderProgram.id;
+
+  return result;
+};
+
 function getHdrShaderProgram(context, shaderProgram) {
   const cachedShader = context.shaderCache.getDerivedShaderProgram(
     shaderProgram,

packages/engine/Source/Scene/FrameState.js

@@ -421,6 +421,37 @@ function FrameState(context, creditDisplay, jobScheduler) {
    * @default 0.0
    */
   this.minimumTerrainHeight = 0.0;
+
+  /**
+   * Whether metadata picking is currently in progress.
+   *
+   * This is set to `true` in the `Picking.pickMetadata` function,
+   * immediately before updating and executing the draw commands,
+   * and set back to `false` immediately afterwards. It will be
+   * used to determine whether the metadata picking draw commands
+   * should be executed, in the `Scene.executeCommand` function.
+   *
+   * @type {boolean}
+   * @default false
+   */
+  this.pickingMetadata = false;
+
+  /**
+   * Metadata picking information.
+   *
+   * This describes the metadata property that is supposed to be picked
+   * in a `Picking.pickMetadata` call.
+   *
+   * This is stored in the frame state and in the metadata picking draw
+   * commands. In the `Scene.updateDerivedCommands` call, it will be
+   * checked whether the instance that is stored in the frame state
+   * is different from the one in the draw command, and if necessary,
+   * the derived commands for metadata picking will be updated based
+   * on this information.
+   *
+   * @type {PickedMetadataInfo|undefined}
+   */
+  this.pickedMetadataInfo = undefined;
 }
 
 /**