suggestion for isDataAvailable, stinky

Author: zilm13

ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/common/helpers/MiscHelpers.java

@@ -21,6 +21,7 @@
 
 import com.google.common.primitives.UnsignedBytes;
 import it.unimi.dsi.fastutil.ints.IntList;
+import java.util.Collection;
 import java.util.List;
 import java.util.Optional;
 import java.util.stream.IntStream;
@@ -344,10 +345,19 @@ public boolean isExecutionEnabled(final BeaconState genericState, final BeaconBl
   }
 
   /**
-   * Performs data availability check (just KZG validation) assuming blobSidecar content (slot,
-   * parent root, commitments) have been previously matched against the block
+   * Performs data availability check for blobSidecars against block, assumes {@link
+   * #verifyKzgProofBlobsBatch(List)} and {@link #validateBlobSidecarsAgainstBlock(List,
+   * BeaconBlock, List)} were already performed
    */
-  public boolean isDataAvailable(final List<BlobSidecar> blobSidecars) {
+  public boolean isDataAvailable(
+      final BeaconBlock block, final Collection<BlobSidecar> verifiedBlobSidecars) {
+    return false;
+  }
+  /**
+   * Performs KZG validation assuming blobSidecar content (slot, parent root, commitments) have been
+   * previously matched against the block
+   */
+  public boolean verifyKzgProofBlobsBatch(final List<BlobSidecar> blobSidecars) {
     return false;
   }
 

ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/blobs/BlobSidecarsAvailabilityChecker.java

@@ -57,6 +57,6 @@ public BlobSidecarsAndValidationResult validateImmediately(
 
   SafeFuture<BlobSidecarsAndValidationResult> getAvailabilityCheckResult();
 
-  /** Only perform the {@link MiscHelpersDeneb#isDataAvailable} check */
+  /** Only perform the {@link MiscHelpersDeneb#verifyKzgProofBlobsBatch(List)} check */
   BlobSidecarsAndValidationResult validateImmediately(List<BlobSidecar> blobSidecars);
 }

ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/helpers/MiscHelpersDeneb.java

@@ -17,8 +17,12 @@
 import static tech.pegasys.teku.spec.config.SpecConfigDeneb.VERSIONED_HASH_VERSION_KZG;
 
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.List;
 import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 import org.apache.tuweni.bytes.Bytes;
 import tech.pegasys.teku.infrastructure.crypto.Hash;
 import tech.pegasys.teku.infrastructure.ssz.SszList;
@@ -34,6 +38,7 @@
 import tech.pegasys.teku.spec.datastructures.blocks.BeaconBlock;
 import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock;
 import tech.pegasys.teku.spec.datastructures.blocks.blockbody.versions.deneb.BeaconBlockBodyDeneb;
+import tech.pegasys.teku.spec.datastructures.networking.libp2p.rpc.BlobIdentifier;
 import tech.pegasys.teku.spec.logic.versions.capella.helpers.MiscHelpersCapella;
 import tech.pegasys.teku.spec.logic.versions.deneb.types.VersionedHash;
 
@@ -58,15 +63,25 @@ private static KZG initKZG(final SpecConfigDeneb config) {
     return kzg;
   }
 
-  /**
-   * <a
-   * href="https://github.com/ethereum/consensus-specs/blob/dev/specs/deneb/fork-choice.md#is_data_available">is_data_available</a>
-   *
-   * <p>In Deneb we don't need to retrieve data, everything is already available via blob sidecars.
-   */
   @Override
-  public boolean isDataAvailable(final List<BlobSidecar> blobSidecars) {
+  public boolean isDataAvailable(
+      final BeaconBlock block, final Collection<BlobSidecar> verifiedBlobSidecars) {
+    final Set<BlobIdentifier> verifiedBlobIdentifiers =
+        verifiedBlobSidecars.stream()
+            .map(
+                blobSidecar ->
+                    new BlobIdentifier(blobSidecar.getBlockRoot(), blobSidecar.getIndex()))
+            .collect(Collectors.toSet());
+    return IntStream.range(
+            0, BeaconBlockBodyDeneb.required(block.getBody()).getBlobKzgCommitments().size())
+        .mapToObj(index -> new BlobIdentifier(block.getRoot(), UInt64.valueOf(index)))
+        .filter(blobIdentifier -> !verifiedBlobIdentifiers.contains(blobIdentifier))
+        .findFirst()
+        .isEmpty();
+  }
 
+  @Override
+  public boolean verifyKzgProofBlobsBatch(final List<BlobSidecar> blobSidecars) {
     final List<Bytes> blobs = new ArrayList<>();
     final List<KZGProof> proofs = new ArrayList<>();
     final List<KZGCommitment> kzgCommitments = new ArrayList<>();
@@ -93,7 +108,9 @@ public VersionedHash kzgCommitmentToVersionedHash(final KZGCommitment kzgCommitm
 
   /**
    * The validation assumes that blockRoot and Slot are already matching. This is guaranteed by
-   * BlobSidecarPool and BlockBlobSidecarsTracker
+   * BlobSidecarPool and BlockBlobSidecarsTracker.
+   *
+   * <p>Doesn't require all blobs to be provided, verifies only provided blobs
    *
    * @param blobSidecars blob sidecars to validate
    * @param block block to validate blob sidecar against

ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ForkChoiceBlobSidecarsAvailabilityChecker.java

@@ -19,7 +19,7 @@
 import com.google.common.base.Suppliers;
 import java.time.Duration;
 import java.util.ArrayList;
-import java.util.Collections;
+import java.util.Collection;
 import java.util.List;
 import java.util.NavigableMap;
 import java.util.Optional;
@@ -136,7 +136,7 @@ private BlobSidecarsAndValidationResult validateBatch(final List<BlobSidecar> bl
       miscHelpers.validateBlobSidecarsAgainstBlock(
           blobSidecars, block, kzgCommitmentsFromBlockSupplier.get());
 
-      if (!miscHelpers.isDataAvailable(blobSidecars)) {
+      if (!miscHelpers.verifyKzgProofBlobsBatch(blobSidecars)) {
         return BlobSidecarsAndValidationResult.invalidResult(blobSidecars);
       }
     } catch (final Exception ex) {
@@ -196,7 +196,11 @@ && isBlockOutsideDataAvailabilityWindow()) {
     }
 
     if (performCompleteValidation) {
-      return Optional.of(result);
+      if (result.isValid()) {
+        return Optional.of(completeValidation());
+      } else {
+        return Optional.of(result);
+      }
     }
 
     // cache partially validated blobs
@@ -308,19 +312,25 @@ private BlobSidecarsAndValidationResult computeFinalValidationResult(
         .getBlobSidecars()
         .forEach(blobSidecar -> validatedBlobSidecars.put(blobSidecar.getIndex(), blobSidecar));
 
-    // let's create the final list of validated BlobSidecars making sure that indices and
-    // final order are consistent
-    final List<BlobSidecar> completeValidatedBlobSidecars = new ArrayList<>();
-    validatedBlobSidecars.forEach(
-        (index, blobSidecar) -> {
-          checkState(
-              index.equals(blobSidecar.getIndex())
-                  && index.equals(UInt64.valueOf(completeValidatedBlobSidecars.size())),
-              "Inconsistency detected during blob sidecar validation");
-          completeValidatedBlobSidecars.add(blobSidecar);
-        });
-
-    if (completeValidatedBlobSidecars.size() < kzgCommitmentsFromBlockSupplier.get().size()) {
+    return completeValidation();
+  }
+
+  private BlobSidecarsAndValidationResult completeValidation() {
+    final BeaconBlock block = blockBlobSidecarsTracker.getBlock().orElseThrow();
+    final MiscHelpers miscHelpers = spec.atSlot(block.getSlot()).miscHelpers();
+    final Collection<BlobSidecar> maybeVerifiedBlobSidecars =
+        validatedBlobSidecars
+            .subMap(
+                UInt64.ZERO,
+                UInt64.valueOf(
+                    BeaconBlockBodyDeneb.required(
+                            blockBlobSidecarsTracker.getBlock().orElseThrow().getBody())
+                        .getBlobKzgCommitments()
+                        .size()))
+            .values();
+    final boolean isDataAvailable = miscHelpers.isDataAvailable(block, maybeVerifiedBlobSidecars);
+
+    if (!isDataAvailable) {
       // we haven't verified enough blobs to match the commitments present in the block
       // this should never happen in practice. If it does, is likely a bug and should be fixed.
       checkState(
@@ -332,8 +342,7 @@ private BlobSidecarsAndValidationResult computeFinalValidationResult(
       return BlobSidecarsAndValidationResult.NOT_REQUIRED;
     }
 
-    return BlobSidecarsAndValidationResult.validResult(
-        Collections.unmodifiableList(completeValidatedBlobSidecars));
+    return BlobSidecarsAndValidationResult.validResult(maybeVerifiedBlobSidecars.stream().toList());
   }
 
   private boolean isBlockOutsideDataAvailabilityWindow() {