feat: implement ViewDU.batchHashTreeRoot, merge PR #392

packages/ssz/src/type/arrayBasic.ts

@@ -5,8 +5,8 @@ import {
   getNodesAtDepth,
   packedNodeRootsToBytes,
   packedRootsBytesToNode,
-  levelAtIndex,
   HashComputationLevel,
+  levelAtIndex,
 } from "@chainsafe/persistent-merkle-tree";
 import {Type, ValueOf, ByteViews} from "./abstract";
 import {BasicType} from "./basic";

packages/ssz/src/type/bitArray.ts

@@ -1,4 +1,4 @@
-import {concatGindices, Gindex, HashComputationLevel, Node, toGindex, Tree} from "@chainsafe/persistent-merkle-tree";
+import {concatGindices, Gindex, Node, toGindex, Tree, HashComputationLevel} from "@chainsafe/persistent-merkle-tree";
 import {fromHexString, toHexString, byteArrayEquals} from "../util/byteArray";
 import {CompositeType, LENGTH_GINDEX} from "./composite";
 import {BitArray} from "../value/bitArray";

packages/ssz/src/type/byteArray.ts

@@ -1,4 +1,12 @@
-import {concatGindices, Gindex, Node, toGindex, Tree} from "@chainsafe/persistent-merkle-tree";
+import {
+  concatGindices,
+  Gindex,
+  Node,
+  toGindex,
+  Tree,
+  HashComputationLevel,
+  getHashComputations,
+} from "@chainsafe/persistent-merkle-tree";
 import {fromHexString, toHexString, byteArrayEquals} from "../util/byteArray";
 import {ByteViews} from "./abstract";
 import {CompositeType, LENGTH_GINDEX} from "./composite";
@@ -36,12 +44,16 @@ export abstract class ByteArrayType extends CompositeType<ByteArray, ByteArray,
     return this.commitViewDU(view);
   }
 
-  // TODO - batch
-  commitViewDU(view: ByteArray): Node {
+  // there is no respective ViewDU for this type
+  commitViewDU(view: ByteArray, hcOffset = 0, hcByLevel: HashComputationLevel[] | null = null): Node {
     const uint8Array = new Uint8Array(this.value_serializedSize(view));
     const dataView = new DataView(uint8Array.buffer, uint8Array.byteOffset, uint8Array.byteLength);
     this.value_serializeToBytes({uint8Array, dataView}, 0, view);
-    return this.tree_deserializeFromBytes({uint8Array, dataView}, 0, uint8Array.length);
+    const node = this.tree_deserializeFromBytes({uint8Array, dataView}, 0, uint8Array.length);
+    if (hcByLevel !== null && node.h0 === null) {
+      getHashComputations(node, hcOffset, hcByLevel);
+    }
+    return node;
   }
 
   cacheOfViewDU(): unknown {

packages/ssz/src/type/optional.ts

@@ -1,4 +1,13 @@
-import {concatGindices, Gindex, merkleizeInto, Node, Tree, zeroNode} from "@chainsafe/persistent-merkle-tree";
+import {
+  concatGindices,
+  Gindex,
+  merkleizeInto,
+  Node,
+  Tree,
+  zeroNode,
+  getHashComputations,
+  HashComputationLevel,
+} from "@chainsafe/persistent-merkle-tree";
 import {Require} from "../util/types";
 import {namedClass} from "../util/named";
 import {Type, ByteViews, JsonPath, JsonPathProp} from "./abstract";
@@ -81,9 +90,12 @@ export class OptionalType<ElementType extends Type<unknown>> extends CompositeTy
   }
 
   // TODO add an OptionalViewDU
-  // TODO - batch
-  commitViewDU(view: ValueOfType<ElementType>): Node {
-    return this.value_toTree(view);
+  commitViewDU(view: ValueOfType<ElementType>, hcOffset = 0, hcByLevel: HashComputationLevel[] | null = null): Node {
+    const node = this.value_toTree(view);
+    if (hcByLevel !== null && node.h0 === null) {
+      getHashComputations(node, hcOffset, hcByLevel);
+    }
+    return node;
   }
 
   // TODO add an OptionalViewDU

packages/ssz/src/type/union.ts

@@ -1,4 +1,13 @@
-import {concatGindices, getNode, Gindex, Node, Tree, merkleizeInto} from "@chainsafe/persistent-merkle-tree";
+import {
+  concatGindices,
+  getNode,
+  Gindex,
+  Node,
+  Tree,
+  merkleizeInto,
+  getHashComputations,
+  HashComputationLevel,
+} from "@chainsafe/persistent-merkle-tree";
 import {Require} from "../util/types";
 import {namedClass} from "../util/named";
 import {Type, ByteViews} from "./abstract";
@@ -112,9 +121,12 @@ export class UnionType<Types extends Type<unknown>[]> extends CompositeType<
     return this.value_toTree(view);
   }
 
-  // TODO - batch
-  commitViewDU(view: ValueOfTypes<Types>): Node {
-    return this.value_toTree(view);
+  commitViewDU(view: ValueOfTypes<Types>, hcOffset = 0, hcByLevel: HashComputationLevel[] | null = null): Node {
+    const node = this.value_toTree(view);
+    if (hcByLevel !== null && node.h0 === null) {
+      getHashComputations(node, hcOffset, hcByLevel);
+    }
+    return node;
   }
 
   value_serializedSize(value: ValueOfTypes<Types>): number {

packages/ssz/src/type/vectorComposite.ts

@@ -1,4 +1,4 @@
-import {HashComputationLevel, Node, Tree} from "@chainsafe/persistent-merkle-tree";
+import {Node, Tree, HashComputationLevel} from "@chainsafe/persistent-merkle-tree";
 import {maxChunksToDepth} from "../util/merkleize";
 import {Require} from "../util/types";
 import {namedClass} from "../util/named";

packages/ssz/src/view/arrayBasic.ts

@@ -1,4 +1,4 @@
-import {getNodesAtDepth, HashComputationLevel, LeafNode, Node, Tree} from "@chainsafe/persistent-merkle-tree";
+import {getNodesAtDepth, LeafNode, Node, Tree, HashComputationLevel} from "@chainsafe/persistent-merkle-tree";
 import {ValueOf} from "../type/abstract";
 import {BasicType} from "../type/basic";
 import {CompositeType} from "../type/composite";

packages/ssz/src/view/arrayComposite.ts

@@ -1,4 +1,4 @@
-import {getNodesAtDepth, HashComputationLevel, Node, toGindexBitstring, Tree} from "@chainsafe/persistent-merkle-tree";
+import {getNodesAtDepth, Node, toGindexBitstring, Tree, HashComputationLevel} from "@chainsafe/persistent-merkle-tree";
 import {ValueOf} from "../type/abstract";
 import {CompositeType, CompositeView, CompositeViewDU} from "../type/composite";
 import {TreeView} from "./abstract";

packages/ssz/src/viewDU/abstract.ts

@@ -65,7 +65,7 @@ export abstract class TreeViewDU<T extends CompositeType<unknown, unknown, unkno
 
   /**
    * The same to hashTreeRoot() but with batch hash computation.
-   * Consumer can provide a HashComputationGroup() to save memory allocation.
+   * Consumer can allocate and reuse a HashComputationGroup() if needed.
    */
   batchHashTreeRoot(hcGroup: HashComputationGroup = new HashComputationGroup()): Uint8Array {
     // in ethereum consensus, the only type goes with TVDU is BeaconState and it's really more efficient to hash the tree in batch
@@ -91,7 +91,7 @@ export abstract class TreeViewDU<T extends CompositeType<unknown, unknown, unkno
   /**
    * Serialize view to binary data.
    * Commits any pending changes before computing the root.
-   * Warning: this calls commit() which evict all pending HashComputations. Consider calling hashTreeRoot() before this
+   * This calls commit() which evict all pending HashComputations. Consider calling hashTreeRoot() before this
    */
   serialize(): Uint8Array {
     this.commit();

packages/ssz/src/viewDU/containerNodeStruct.ts

@@ -1,4 +1,4 @@
-import {Node} from "@chainsafe/persistent-merkle-tree";
+import {Node, HashComputationLevel} from "@chainsafe/persistent-merkle-tree";
 import {Type, ValueOf} from "../type/abstract";
 import {isCompositeType} from "../type/composite";
 import {BranchNodeStruct} from "../branchNodeStruct";
@@ -32,35 +32,22 @@ export class ContainerNodeStructTreeViewDU<Fields extends Record<string, Type<un
   }
 
   /**
-   * This ViewDU does not support batch hash by default so we need to compute root immediately.
-   * Otherwise consumers may call commit() multiple times and not able to compute hashTreeRoot().
+   * There are 2 cases:
+   * - normal commit() or hashTreeRoot(): hcByLevel is null, no need to compute root
+   * - batchHashTreeRoot(): hcByLevel is not null, need to compute root because this does not support HashComputation
    */
-  commit(): void {
-    if (this.valueChanged === null) {
-      this._rootNode.rootHashObject;
-      return;
-    }
-
-    const value = this.valueChanged;
-    this.valueChanged = null;
-
-    this._rootNode = this.type.value_toTree(value) as BranchNodeStruct<ValueOfFields<Fields>>;
-    this._rootNode.rootHashObject;
-  }
+  commit(_?: number, hcByLevel: HashComputationLevel[] | null = null): void {
+    if (this.valueChanged !== null) {
+      const value = this.valueChanged;
+      this.valueChanged = null;
 
-  /**
-   * Same to commit() without hash, allow to do the batch hash at consumer side, like in ListValidatorViewDU
-   * of ethereum consensus node.
-   */
-  commitNoHash(): void {
-    if (this.valueChanged === null) {
-      return;
+      this._rootNode = this.type.value_toTree(value) as BranchNodeStruct<ValueOfFields<Fields>>;
     }
 
-    const value = this.valueChanged;
-    this.valueChanged = null;
-
-    this._rootNode = this.type.value_toTree(value) as BranchNodeStruct<ValueOfFields<Fields>>;
+    if (this._rootNode.h0 === null && hcByLevel !== null) {
+      // consumer is batchHashTreeRoot()
+      this._rootNode.rootHashObject;
+    }
   }
 
   protected clearCache(): void {

packages/ssz/test/lodestarTypes/phase0/viewDU/listValidator.ts

@@ -94,7 +94,8 @@ export class ListValidatorTreeViewDU extends ListCompositeTreeViewDU<ValidatorNo
           const indexInBatch = (i - j) % PARALLEL_FACTOR;
           const viewChanged = viewsChanged.get(viewIndex);
           if (viewChanged) {
-            viewChanged.commitNoHash();
+            // should not have any params here in order not to compute root
+            viewChanged.commit();
             const branchNodeStruct = viewChanged.node;
             byteArrayIntoHashObject(validatorRoots[indexInBatch], 0, branchNodeStruct);
             nodesChanged.push({index: viewIndex, node: viewChanged.node});