NPU sample

common/utils.js

@@ -49,9 +49,20 @@ export async function buildConstantByNpy(builder, url) {
   const dimensions = npArray.shape;
   const type = dataTypeMap.get(npArray.dataType).type;
   const TypedArrayConstructor = dataTypeMap.get(npArray.dataType).array;
-  const dataView = new Uint8Array(npArray.data.buffer);
-  const dataView2 = dataView.slice();
-  const typedArray = new TypedArrayConstructor(dataView2.buffer);
+  const typedArray = new TypedArrayConstructor(sizeOfShape(dimensions));
+  const dataView = new DataView(npArray.data.buffer);
+  // const dataView2 = dataView.slice();
+  // const typedArray = new TypedArrayConstructor(dataView2.buffer);
+  // return builder.constant({dataType: type, type, dimensions}, typedArray);
+  const littleEndian = npArray.byteOrder === '<';
+  let getFuncName = `get` + type[0].toUpperCase() + type.substr(1);
+  if (type == 'float16') {
+    getFuncName = `getUint16`;
+  }
+  for (let i = 0; i < sizeOfShape(dimensions); ++i) {
+    typedArray[i] = dataView[getFuncName](
+        i * TypedArrayConstructor.BYTES_PER_ELEMENT, littleEndian);
+  }
   return builder.constant({dataType: type, type, dimensions}, typedArray);
 }
 
@@ -86,6 +97,102 @@ export function stopCameraStream(id, stream) {
   }
 }
 
+// ref: http://stackoverflow.com/questions/32633585/how-do-you-convert-to-half-floats-in-javascript
+const toHalf = (function() {
+
+  var floatView = new Float32Array(1);
+  var int32View = new Int32Array(floatView.buffer);
+
+  /* This method is faster than the OpenEXR implementation (very often
+   * used, eg. in Ogre), with the additional benefit of rounding, inspired
+   * by James Tursa?s half-precision code. */
+  return function toHalf(val) {
+
+    floatView[0] = val;
+    var x = int32View[0];
+
+    var bits = (x >> 16) & 0x8000; /* Get the sign */
+    var m = (x >> 12) & 0x07ff; /* Keep one extra bit for rounding */
+    var e = (x >> 23) & 0xff; /* Using int is faster here */
+
+    /* If zero, or denormal, or exponent underflows too much for a denormal
+     * half, return signed zero. */
+    if (e < 103) {
+      return bits;
+    }
+
+    /* If NaN, return NaN. If Inf or exponent overflow, return Inf. */
+    if (e > 142) {
+      bits |= 0x7c00;
+      /* If exponent was 0xff and one mantissa bit was set, it means NaN,
+       * not Inf, so make sure we set one mantissa bit too. */
+      bits |= ((e == 255) ? 0 : 1) && (x & 0x007fffff);
+      return bits;
+    }
+
+    /* If exponent underflows but not too much, return a denormal */
+    if (e < 113) {
+      m |= 0x0800;
+      /* Extra rounding may overflow and set mantissa to 0 and exponent
+       * to 1, which is OK. */
+      bits |= (m >> (114 - e)) + ((m >> (113 - e)) & 1);
+      return bits;
+    }
+
+    bits |= ((e - 112) << 10) | (m >> 1);
+    /* Extra rounding. An overflow will set mantissa to 0 and increment
+     * the exponent, which is OK. */
+    bits += m & 1;
+    return bits;
+  };
+
+})();
+
+// This function converts a Float16 stored as the bits of a Uint16 into a Javascript Number.
+// Adapted from: https://gist.github.com/martinkallman/5049614
+// input is a Uint16 (eg, new Uint16Array([value])[0])
+
+export function float16ToNumber(input) {
+  // Create a 32 bit DataView to store the input
+  const arr = new ArrayBuffer(4);
+  const dv = new DataView(arr);
+
+  // Set the Float16 into the last 16 bits of the dataview
+  // So our dataView is [00xx]
+  dv.setUint16(2, input, false);
+
+  // Get all 32 bits as a 32 bit integer
+  // (JS bitwise operations are performed on 32 bit signed integers)
+  const asInt32 = dv.getInt32(0, false);
+
+  // All bits aside from the sign
+  let rest = asInt32 & 0x7fff;
+  // Sign bit
+  let sign = asInt32 & 0x8000;
+  // Exponent bits
+  const exponent = asInt32 & 0x7c00;
+
+  // Shift the non-sign bits into place for a 32 bit Float
+  rest <<= 13;
+  // Shift the sign bit into place for a 32 bit Float
+  sign <<= 16;
+
+  // Adjust bias
+  // https://en.wikipedia.org/wiki/Half-precision_floating-point_format#Exponent_encoding
+  rest += 0x38000000;
+  // Denormals-as-zero
+  rest = (exponent === 0 ? 0 : rest);
+  // Re-insert sign bit
+  rest |= sign;
+
+  // Set the adjusted float32 (stored as int32) back into the dataview
+  dv.setInt32(0, rest, false);
+
+  // Get it back out as a float32 (which js will convert to a Number)
+  const asFloat32 = dv.getFloat32(0, false);
+
+  return asFloat32;
+}
 /**
  * This method is used to covert input element to tensor data.
  * @param {Object} inputElement, an object of HTML [<img> | <video>] element.
@@ -121,9 +228,16 @@ export function stopCameraStream(id, stream) {
  * @return {Object} tensor, an object of input tensor.
  */
 export function getInputTensor(inputElement, inputOptions) {
+  const dataType = inputOptions.dataType || 'float32';
   const inputDimensions = inputOptions.inputDimensions;
-  const tensor = new Float32Array(
+  let tensor;
+  if (dataType === 'float16') {
+    tensor = new Uint16Array(
       inputDimensions.slice(1).reduce((a, b) => a * b));
+  } else {
+    tensor = new Float32Array(
+        inputDimensions.slice(1).reduce((a, b) => a * b));
+  }
 
   inputElement.width = inputElement.videoWidth ||
       inputElement.naturalWidth;
@@ -180,11 +294,11 @@ export function getInputTensor(inputElement, inputOptions) {
           value = pixels[h * width * imageChannels + w * imageChannels + c];
         }
         if (inputLayout === 'nchw') {
-          tensor[c * width * height + h * width + w] =
-              (value - mean[c]) / std[c];
+          tensor[c * width * height + h * width + w] = dataType === 'float16' ?
+              toHalf((value - mean[c]) / std[c]) : (value - mean[c]) / std[c];
         } else {
-          tensor[h * width * channels + w * channels + c] =
-              (value - mean[c]) / std[c];
+          tensor[h * width * channels + w * channels + c] = dataType === 'float16' ?
+              toHalf((value - mean[c]) / std[c]) : (value - mean[c]) / std[c];
         }
       }
     }

image_classification/efficientnet_fp16_nchw.js

@@ -0,0 +1,161 @@
+'use strict';
+
+import {buildConstantByNpy} from '../common/utils.js';
+
+// ResNet50 V1 model with 'nchw' input layout
+export class EfficientNetFP16Nchw {
+  constructor() {
+    this.context_ = null;
+    this.builder_ = null;
+    this.graph_ = null;
+    this.weightsUrl_ = './weights/efficientnet_fp16_nchw_optimized/';
+    this.inputOptions = {
+      mean: [0.485, 0.456, 0.406],
+      std: [0.229, 0.224, 0.225],
+      norm: true,
+      inputLayout: 'nchw',
+      labelUrl: './labels/labels1000.txt',
+      inputDimensions: [1, 3, 224, 224],
+      dataType: 'float16',
+    };
+    this.outputDimensions = [1, 1000];
+  }
+
+  async buildConv_(input, name, blockName, clip = false, options = {}) {
+    let prefix = '';
+    if (blockName !== '') {
+      prefix = this.weightsUrl_ + 'block' + blockName + '_conv' +
+          name;
+    } else {
+      prefix = this.weightsUrl_ + 'conv' + name;
+    }
+    const weightName = prefix + '_w.npy';
+    const weight = await buildConstantByNpy(this.builder_, weightName);
+
+    const biasName = prefix + '_b.npy';
+    options.bias = await buildConstantByNpy(this.builder_, biasName);
+    if (clip){
+      return this.builder_.clamp(
+        this.builder_.conv2d(input, weight, options),
+        {minValue: 0, maxValue: 6});
+    }
+    return this.builder_.conv2d(input, weight, options);
+  }
+
+  async buildGemm_(input, name) {
+    const prefix = this.weightsUrl_ + 'dense' + name;
+    const weightName = prefix + '_w.npy';
+    const weight = await buildConstantByNpy(this.builder_, weightName);
+    const biasName = prefix + '_b.npy';
+    const bias = await buildConstantByNpy(this.builder_, biasName);
+    const options =
+        {c: this.builder_.reshape(bias, [1, 1000]), bTranspose: true};
+    return this.builder_.gemm(input, weight, options);
+  }
+
+  async buildBottlenect_(input, blockName, group, pad = 1) {
+    const conv1 = await this.buildConv_(input, '0', blockName, true);
+    const conv2 = await this.buildConv_(
+      conv1, '1', blockName, true, { groups: group, padding: [pad, pad, pad, pad] });
+    const conv3 = await this.buildConv_(conv2, '2', blockName);
+    return this.builder_.add(conv3, input);
+  }
+
+  async load(contextOptions) {
+    this.context_ = await navigator.ml.createContext(contextOptions);
+    this.builder_ = new MLGraphBuilder(this.context_);
+    const data = this.builder_.input('input', { dataType: this.inputOptions.dataType, dimensions: this.inputOptions.inputDimensions });
+    // Block 0
+    const conv1 = await this.buildConv_(
+      data, '0', '0', true, { padding: [0, 1, 0, 1], strides: [2, 2] });
+    const conv2 = await this.buildConv_(conv1, '1', '0', true, { groups: 32, padding: [1, 1, 1, 1] });
+    const conv3 = await this.buildConv_(conv2, '2', '0');
+
+    // Block 1
+    const conv4 = await this.buildConv_(conv3, '0', '1', true);
+    const conv5 = await this.buildConv_(conv4, '1', '1', true, { groups: 144, padding: [0, 1, 0, 1], strides: [2, 2] });
+    const conv6 = this.buildConv_(conv5, '2', '1');
+
+    // Block 2~4
+    const bottleneck2 = await this.buildBottlenect_(conv6, '2', 192);
+    const bottleneck3 = await this.buildBottlenect_(bottleneck2, '3', 192);
+    const bottleneck4 = await this.buildBottlenect_(bottleneck3, '4', 192);
+
+    // Block 5
+    const conv7 = await this.buildConv_(bottleneck4, '0', '5', true);
+    const conv8 = await this.buildConv_(conv7, '1', '5', true, { groups: 192, padding: [1, 2, 1, 2], strides: [2, 2] });
+    const conv9 = await this.buildConv_(conv8, '2', '5');
+
+    // Block 6~8
+    const bottleneck6 = await this.buildBottlenect_(conv9, '6', 336, 2);
+    const bottleneck7 = await this.buildBottlenect_(bottleneck6, '7', 336, 2);
+    const bottleneck8 = await this.buildBottlenect_(bottleneck7, '8', 336, 2);
+
+    // Block 9
+    const conv10 = await this.buildConv_(bottleneck8, '0', '9', true);
+    const conv11 = await this.buildConv_(conv10, '1', '9', true, { groups: 336, padding: [0, 1, 0, 1], strides: [2, 2] });
+    const conv12 = this.buildConv_(conv11, '2', '9');
+
+    // Block 10~14
+    const bottleneck10 = await this.buildBottlenect_(conv12, '10', 672);
+    const bottleneck11 = await this.buildBottlenect_(bottleneck10, '11', 672);
+    const bottleneck12 = await this.buildBottlenect_(bottleneck11, '12', 672);
+    const bottleneck13 = await this.buildBottlenect_(bottleneck12, '13', 672);
+    const bottleneck14 = await this.buildBottlenect_(bottleneck13, '14', 672);
+
+    // Block 15
+    const conv13 = await this.buildConv_(bottleneck14, '0', '15', true);
+    const conv14 = await this.buildConv_(conv13, '1', '15', true, { groups: 672, padding: [2, 2, 2, 2] });
+    const conv15 = this.buildConv_(conv14, '2', '15');
+
+    // Block 16~20
+    const bottleneck16 = await this.buildBottlenect_(conv15, '16', 960, 2);
+    const bottleneck17 = await this.buildBottlenect_(bottleneck16, '17', 960, 2);
+    const bottleneck18 = await this.buildBottlenect_(bottleneck17, '18', 960, 2);
+    const bottleneck19 = await this.buildBottlenect_(bottleneck18, '19', 960, 2);
+    const bottleneck20 = await this.buildBottlenect_(bottleneck19, '20', 960, 2);
+
+    // Block 21
+    const conv16 = await this.buildConv_(bottleneck20, '0', '21', true);
+    const conv17 = await this.buildConv_(conv16, '1', '21', true, { groups: 960, padding: [1, 2, 1, 2], strides: [2, 2] });
+    const conv18 = await this.buildConv_(conv17, '2', '21');
+
+    // Block 22~28
+    const bottleneck22 = await this.buildBottlenect_(conv18, '22', 1632, 2);
+    const bottleneck23 = await this.buildBottlenect_(bottleneck22, '23', 1632, 2);
+    const bottleneck24 = await this.buildBottlenect_(bottleneck23, '24', 1632, 2);
+    const bottleneck25 = await this.buildBottlenect_(bottleneck24, '25', 1632, 2);
+    const bottleneck26 = await this.buildBottlenect_(bottleneck25, '26', 1632, 2);
+    const bottleneck27 = await this.buildBottlenect_(bottleneck26, '27', 1632, 2);
+    const bottleneck28 = await this.buildBottlenect_(bottleneck27, '28', 1632, 2);
+
+    // Block 29
+    const conv19 = await this.buildConv_(bottleneck28, '0', '29', true);
+    const conv20 = await this.buildConv_(conv19, '1', '29', true, { groups: 1632, padding: [1, 1, 1, 1] });
+    const conv21 = await this.buildConv_(conv20, '2', '29');
+
+    const conv22 = this.buildConv_(conv21, '2', '', true);
+    const pool1 = await this.builder_.averagePool2d(conv22);
+    const reshape = await this.builder_.reshape(pool1, [1, 1028]);
+    return this.buildGemm_(reshape, '0');
+  }
+
+  async build(outputOperand) {
+    this.graph_ = await this.builder_.build({'output': outputOperand});
+  }
+
+  // Release the constant tensors of a model
+  dispose() {
+    // dispose() is only available in webnn-polyfill
+    if (this.graph_ !== null && 'dispose' in this.graph_) {
+      this.graph_.dispose();
+    }
+  }
+
+  async compute(inputBuffer, outputBuffer) {
+    const inputs = {'input': inputBuffer};
+    const outputs = {'output': outputBuffer};
+    const results = await this.context_.compute(this.graph_, inputs, outputs);
+    return results;
+  }
+}

image_classification/index.html

@@ -31,17 +31,20 @@
           </div>
           <div class="col-md-auto">
             <div class="btn-group-toggle" data-toggle="buttons" id="backendBtns">
-              <label class="btn btn-outline-info custom" name="polyfill">
+              <!-- <label class="btn btn-outline-info custom" name="polyfill">
                 <input type="radio" name="backend" id="polyfill_cpu" autocomplete="off">Wasm (CPU)
               </label>
               <label class="btn btn-outline-info custom" name="polyfill">
                 <input type="radio" name="backend" id="polyfill_gpu" autocomplete="off">WebGL (GPU)
-              </label>
+              </label> -->
               <label class="btn btn-outline-info custom" name="webnn">
                 <input type="radio" name="backend" id="webnn_cpu" autocomplete="off">WebNN (CPU)
               </label>
               <label class="btn btn-outline-info custom" name="webnn">
-                <input type="radio" name="backend" id="webnn_gpu" autocomplete="off">WebNN (GPU)
+                <input type="radio" name="backend" id="webnn_gpu" autocomplete="off" checked>WebNN (GPU)
+              </label>
+              <label class="btn btn-outline-info custom" name="webnn">
+                <input type="radio" name="backend" id="webnn_npu" autocomplete="off">WebNN (NPU)
               </label>
             </div>
           </div>
@@ -55,8 +58,8 @@
               <label class="btn btn-outline-info" id='nchw-label'>
                 <input type="radio" name="layout" id="nchw" autocomplete="off">NCHW
               </label>
-              <label class="btn btn-outline-info btn-sm active">
-                <input type="radio" name="layout" id="nhwc" autocomplete="off" checked>NHWC
+              <label class="btn btn-outline-info btn-sm">
+                <input type="radio" name="layout" id="nhwc" autocomplete="off">NHWC
               </label>
             </div>
           </div>
@@ -67,6 +70,18 @@
           </div>
           <div class="col-md-auto">
             <div class="btn-group-toggle" data-toggle="buttons" id="modelBtns">
+              <label class="btn btn-outline-info">
+                <input type="radio" name="model" id="mobilenetv27fp16" autocomplete="off">MobileNet V2 FP16
+              </label>
+              <label class="btn btn-outline-info">
+                <input type="radio" name="model" id="squeezenetfp16" autocomplete="off">SqueezeNet 1.0 FP16
+              </label>
+              <label class="btn btn-outline-info">
+                <input type="radio" name="model" id="resnet50v1fp16" autocomplete="off">ResNet 50 V1 FP16
+              </label>
+              <label class="btn btn-outline-info">
+                <input type="radio" name="model" id="efficientnetfp16" autocomplete="off">Efficientnet FP16
+              </label>
               <label class="btn btn-outline-info">
                 <input type="radio" name="model" id="mobilenet" autocomplete="off">MobileNet V2
               </label>
@@ -213,6 +228,9 @@ <h2 class="text-uppercase text-info">No model selected</h2>
   <script src="https://cdn.jsdelivr.net/npm/popper.js@1.16.1/dist/umd/popper.min.js"
     integrity="sha384-9/reFTGAW83EW2RDu2S0VKaIzap3H66lZH81PoYlFhbGU+6BZp6G7niu735Sk7lN"
     crossorigin="anonymous"></script>
+  <script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs@3.9.0/dist/tf.min.js"
+    integrity="sha256-28ZvjeNGrGNEIj9/2D8YAPE6Vm5JSvvDs+LI4ED31x8="
+    crossorigin="anonymous"></script>
   <script src="https://stackpath.bootstrapcdn.com/bootstrap/4.5.2/js/bootstrap.min.js"
     integrity="sha384-B4gt1jrGC7Jh4AgTPSdUtOBvfO8shuf57BaghqFfPlYxofvL8/KUEfYiJOMMV+rV"
     crossorigin="anonymous"></script>