From 9cc745a54080ddc1441e839f359f573987325266 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Sat, 4 Nov 2023 15:24:08 +0100 Subject: [PATCH] update Deneb for blob sidecar inclusion proofs `BlobSidecar` is no longer signed, instead use Merkle proof to link blobs with block. - https://github.com/ethereum/consensus-specs/pull/3531 Associated beacon-API / builder-specs still TBD; minimal changes done to compile in similar style to previous spec, but not standardized yet. - https://github.com/ethereum/beacon-APIs/pull/369 - https://github.com/ethereum/builder-specs/pull/90 --- AllTests-mainnet.md | 5 +- ConsensusSpecPreset-mainnet.md | 23 ++- ConsensusSpecPreset-minimal.md | 26 ++- beacon_chain/beacon_chain_db.nim | 3 +- .../blob_quarantine.nim | 23 ++- .../gossip_processing/eth2_processor.nim | 32 ++- .../gossip_processing/gossip_validation.nim | 193 ++++++++++++------ beacon_chain/networking/eth2_network.nim | 2 +- beacon_chain/nimbus_beacon_node.nim | 8 +- beacon_chain/rpc/rest_beacon_api.nim | 42 ++-- beacon_chain/rpc/rest_validator_api.nim | 21 +- beacon_chain/spec/datatypes/base.nim | 2 +- beacon_chain/spec/datatypes/constants.nim | 3 - beacon_chain/spec/datatypes/deneb.nim | 73 ++++--- .../eth2_apis/eth2_rest_serialization.nim | 43 +++- beacon_chain/spec/eth2_apis/rest_types.nim | 6 +- beacon_chain/spec/forks.nim | 17 +- beacon_chain/spec/helpers.nim | 48 +++++ beacon_chain/spec/mev/deneb_mev.nim | 15 +- beacon_chain/spec/signatures.nim | 30 --- beacon_chain/sync/request_manager.nim | 8 +- beacon_chain/sync/sync_manager.nim | 9 +- beacon_chain/sync/sync_queue.nim | 4 +- .../validator_client/block_service.nim | 44 ++-- beacon_chain/validators/beacon_validators.nim | 107 ++-------- beacon_chain/validators/message_router.nim | 30 ++- .../validators/message_router_mev.nim | 5 +- beacon_chain/validators/validator_pool.nim | 15 -- .../consensus_spec_tests_preset.nim | 1 + .../test_fixture_ssz_consensus_objects.nim | 1 - .../test_fixture_merkle_proof.nim | 74 +++++++ tests/test_beacon_chain_db.nim | 20 +- tests/test_message_signatures.nim | 30 --- tests/test_rest_json_serialization.nim | 2 +- tests/test_signing_node.nim | 2 +- tests/test_sync_manager.nim | 12 +- vendor/nim-eth2-scenarios | 2 +- 37 files changed, 538 insertions(+), 443 deletions(-) create mode 100644 tests/consensus_spec/test_fixture_merkle_proof.nim diff --git a/AllTests-mainnet.md b/AllTests-mainnet.md index 007d509785..a67913aeb3 100644 --- a/AllTests-mainnet.md +++ b/AllTests-mainnet.md @@ -358,7 +358,6 @@ OK: 4/4 Fail: 0/4 Skip: 0/4 ```diff + Aggregate and proof signatures OK + Attestation signatures OK -+ Blob sidecar signatures OK + Deposit signatures OK + Slot signatures OK + Sync committee message signatures OK @@ -366,7 +365,7 @@ OK: 4/4 Fail: 0/4 Skip: 0/4 + Sync committee signed contribution and proof signatures OK + Voluntary exit signatures OK ``` -OK: 9/9 Fail: 0/9 Skip: 0/9 +OK: 8/8 Fail: 0/8 Skip: 0/8 ## Network metadata ```diff + goerli OK @@ -715,4 +714,4 @@ OK: 2/2 Fail: 0/2 Skip: 0/2 OK: 9/9 Fail: 0/9 Skip: 0/9 ---TOTAL--- -OK: 404/409 Fail: 0/409 Skip: 5/409 +OK: 403/408 Fail: 0/408 Skip: 5/408 diff --git a/ConsensusSpecPreset-mainnet.md b/ConsensusSpecPreset-mainnet.md index a246091457..b1366d9dda 100644 --- a/ConsensusSpecPreset-mainnet.md +++ b/ConsensusSpecPreset-mainnet.md @@ -2258,7 +2258,6 @@ OK: 34/34 Fail: 0/34 Skip: 0/34 + Testing SignedBLSToExecutionChange OK + Testing SignedBeaconBlock OK + Testing SignedBeaconBlockHeader OK -+ Testing SignedBlobSidecar OK + Testing SignedContributionAndProof OK + Testing SignedVoluntaryExit OK + Testing SigningData OK @@ -2271,7 +2270,7 @@ OK: 34/34 Fail: 0/34 Skip: 0/34 + Testing VoluntaryExit OK + Testing Withdrawal OK ``` -OK: 49/49 Fail: 0/49 Skip: 0/49 +OK: 48/48 Fail: 0/48 Skip: 0/48 ## EF - Deneb - Sanity - Blocks [Preset: mainnet] ```diff + [Invalid] EF - Deneb - Sanity - Blocks - invalid_all_zeroed_sig [Preset: mainnet] OK @@ -2411,6 +2410,11 @@ OK: 4/4 Fail: 0/4 Skip: 0/4 + Light client - Single merkle proof - mainnet/deneb/light_client/single_merkle_proof/Beacon OK ``` OK: 14/14 Fail: 0/14 Skip: 0/14 +## EF - Merkle proof [Preset: mainnet] +```diff ++ Merkle proof - Single merkle proof - mainnet/deneb/merkle_proof/single_merkle_proof/Beacon OK +``` +OK: 1/1 Fail: 0/1 Skip: 0/1 ## EF - Phase 0 - Epoch Processing - Effective balance updates [Preset: mainnet] ```diff + Effective balance updates - effective_balance_hysteresis [Preset: mainnet] OK @@ -2825,6 +2829,8 @@ OK: 40/40 Fail: 0/40 Skip: 0/40 + ForkChoice - mainnet/altair/fork_choice/get_head/pyspec_tests/proposer_boost_correct_head OK + ForkChoice - mainnet/altair/fork_choice/get_head/pyspec_tests/shorter_chain_but_heavier_we OK + ForkChoice - mainnet/altair/fork_choice/get_head/pyspec_tests/split_tie_breaker_no_attesta OK ++ ForkChoice - mainnet/altair/fork_choice/get_proposer_head/pyspec_tests/basic_is_head_root OK ++ ForkChoice - mainnet/altair/fork_choice/get_proposer_head/pyspec_tests/basic_is_parent_roo OK + ForkChoice - mainnet/altair/fork_choice/on_block/pyspec_tests/basic OK + ForkChoice - mainnet/altair/fork_choice/on_block/pyspec_tests/on_block_bad_parent_root OK ForkChoice - mainnet/altair/fork_choice/on_block/pyspec_tests/on_block_future_block Skip @@ -2842,6 +2848,8 @@ OK: 40/40 Fail: 0/40 Skip: 0/40 + ForkChoice - mainnet/bellatrix/fork_choice/get_head/pyspec_tests/proposer_boost_correct_he OK + ForkChoice - mainnet/bellatrix/fork_choice/get_head/pyspec_tests/shorter_chain_but_heavier OK + ForkChoice - mainnet/bellatrix/fork_choice/get_head/pyspec_tests/split_tie_breaker_no_atte OK ++ ForkChoice - mainnet/bellatrix/fork_choice/get_proposer_head/pyspec_tests/basic_is_head_ro OK ++ ForkChoice - mainnet/bellatrix/fork_choice/get_proposer_head/pyspec_tests/basic_is_parent_ OK + ForkChoice - mainnet/bellatrix/fork_choice/on_block/pyspec_tests/basic OK + ForkChoice - mainnet/bellatrix/fork_choice/on_block/pyspec_tests/on_block_bad_parent_root OK ForkChoice - mainnet/bellatrix/fork_choice/on_block/pyspec_tests/on_block_future_block Skip @@ -2852,6 +2860,7 @@ OK: 40/40 Fail: 0/40 Skip: 0/40 ForkChoice - mainnet/bellatrix/fork_choice/on_merge_block/pyspec_tests/block_lookup_failed Skip ForkChoice - mainnet/bellatrix/fork_choice/on_merge_block/pyspec_tests/too_early_for_merge Skip ForkChoice - mainnet/bellatrix/fork_choice/on_merge_block/pyspec_tests/too_late_for_merge Skip ++ ForkChoice - mainnet/bellatrix/fork_choice/should_override_forkchoice_update/pyspec_tests/ OK + ForkChoice - mainnet/capella/fork_choice/ex_ante/pyspec_tests/ex_ante_attestations_is_grea OK + ForkChoice - mainnet/capella/fork_choice/ex_ante/pyspec_tests/ex_ante_sandwich_with_boost_ OK + ForkChoice - mainnet/capella/fork_choice/ex_ante/pyspec_tests/ex_ante_sandwich_with_honest OK @@ -2863,12 +2872,15 @@ OK: 40/40 Fail: 0/40 Skip: 0/40 + ForkChoice - mainnet/capella/fork_choice/get_head/pyspec_tests/proposer_boost_correct_head OK + ForkChoice - mainnet/capella/fork_choice/get_head/pyspec_tests/shorter_chain_but_heavier_w OK + ForkChoice - mainnet/capella/fork_choice/get_head/pyspec_tests/split_tie_breaker_no_attest OK ++ ForkChoice - mainnet/capella/fork_choice/get_proposer_head/pyspec_tests/basic_is_head_root OK ++ ForkChoice - mainnet/capella/fork_choice/get_proposer_head/pyspec_tests/basic_is_parent_ro OK + ForkChoice - mainnet/capella/fork_choice/on_block/pyspec_tests/basic OK + ForkChoice - mainnet/capella/fork_choice/on_block/pyspec_tests/on_block_bad_parent_root OK ForkChoice - mainnet/capella/fork_choice/on_block/pyspec_tests/on_block_future_block Skip + ForkChoice - mainnet/capella/fork_choice/on_block/pyspec_tests/proposer_boost OK + ForkChoice - mainnet/capella/fork_choice/on_block/pyspec_tests/proposer_boost_is_first_blo OK + ForkChoice - mainnet/capella/fork_choice/on_block/pyspec_tests/proposer_boost_root_same_sl OK ++ ForkChoice - mainnet/capella/fork_choice/should_override_forkchoice_update/pyspec_tests/sh OK + ForkChoice - mainnet/deneb/fork_choice/ex_ante/pyspec_tests/ex_ante_attestations_is_greate OK + ForkChoice - mainnet/deneb/fork_choice/ex_ante/pyspec_tests/ex_ante_sandwich_with_boost_no OK + ForkChoice - mainnet/deneb/fork_choice/ex_ante/pyspec_tests/ex_ante_sandwich_with_honest_a OK @@ -2880,6 +2892,8 @@ OK: 40/40 Fail: 0/40 Skip: 0/40 + ForkChoice - mainnet/deneb/fork_choice/get_head/pyspec_tests/proposer_boost_correct_head OK + ForkChoice - mainnet/deneb/fork_choice/get_head/pyspec_tests/shorter_chain_but_heavier_wei OK + ForkChoice - mainnet/deneb/fork_choice/get_head/pyspec_tests/split_tie_breaker_no_attestat OK ++ ForkChoice - mainnet/deneb/fork_choice/get_proposer_head/pyspec_tests/basic_is_head_root OK ++ ForkChoice - mainnet/deneb/fork_choice/get_proposer_head/pyspec_tests/basic_is_parent_root OK + ForkChoice - mainnet/deneb/fork_choice/on_block/pyspec_tests/basic OK + ForkChoice - mainnet/deneb/fork_choice/on_block/pyspec_tests/invalid_data_unavailable OK + ForkChoice - mainnet/deneb/fork_choice/on_block/pyspec_tests/invalid_incorrect_proof OK @@ -2891,8 +2905,9 @@ OK: 40/40 Fail: 0/40 Skip: 0/40 + ForkChoice - mainnet/deneb/fork_choice/on_block/pyspec_tests/proposer_boost_is_first_block OK + ForkChoice - mainnet/deneb/fork_choice/on_block/pyspec_tests/proposer_boost_root_same_slot OK + ForkChoice - mainnet/deneb/fork_choice/on_block/pyspec_tests/simple_blob_data OK ++ ForkChoice - mainnet/deneb/fork_choice/should_override_forkchoice_update/pyspec_tests/shou OK ``` -OK: 69/77 Fail: 0/77 Skip: 8/77 +OK: 80/88 Fail: 0/88 Skip: 8/88 ## Sync ```diff + Sync - mainnet/bellatrix/sync/optimistic/pyspec_tests/from_syncing_to_invalid OK @@ -2902,4 +2917,4 @@ OK: 69/77 Fail: 0/77 Skip: 8/77 OK: 3/3 Fail: 0/3 Skip: 0/3 ---TOTAL--- -OK: 2336/2344 Fail: 0/2344 Skip: 8/2344 +OK: 2347/2355 Fail: 0/2355 Skip: 8/2355 diff --git a/ConsensusSpecPreset-minimal.md b/ConsensusSpecPreset-minimal.md index decf1beca7..141a6edb54 100644 --- a/ConsensusSpecPreset-minimal.md +++ b/ConsensusSpecPreset-minimal.md @@ -2355,7 +2355,6 @@ OK: 34/34 Fail: 0/34 Skip: 0/34 + Testing SignedBLSToExecutionChange OK + Testing SignedBeaconBlock OK + Testing SignedBeaconBlockHeader OK -+ Testing SignedBlobSidecar OK + Testing SignedContributionAndProof OK + Testing SignedVoluntaryExit OK + Testing SigningData OK @@ -2368,7 +2367,7 @@ OK: 34/34 Fail: 0/34 Skip: 0/34 + Testing VoluntaryExit OK + Testing Withdrawal OK ``` -OK: 49/49 Fail: 0/49 Skip: 0/49 +OK: 48/48 Fail: 0/48 Skip: 0/48 ## EF - Deneb - Sanity - Blocks [Preset: minimal] ```diff + [Invalid] EF - Deneb - Sanity - Blocks - invalid_all_zeroed_sig [Preset: minimal] OK @@ -2551,6 +2550,11 @@ OK: 20/20 Fail: 0/20 Skip: 0/20 + Light client - Update ranking - minimal/deneb/light_client/update_ranking/pyspec_tests/upd OK ``` OK: 4/4 Fail: 0/4 Skip: 0/4 +## EF - Merkle proof [Preset: minimal] +```diff ++ Merkle proof - Single merkle proof - minimal/deneb/merkle_proof/single_merkle_proof/Beacon OK +``` +OK: 1/1 Fail: 0/1 Skip: 0/1 ## EF - Phase 0 - Epoch Processing - Effective balance updates [Preset: minimal] ```diff + Effective balance updates - effective_balance_hysteresis [Preset: minimal] OK @@ -2977,6 +2981,8 @@ OK: 45/45 Fail: 0/45 Skip: 0/45 + ForkChoice - minimal/altair/fork_choice/get_head/pyspec_tests/split_tie_breaker_no_attesta OK + ForkChoice - minimal/altair/fork_choice/get_head/pyspec_tests/voting_source_beyond_two_epo OK + ForkChoice - minimal/altair/fork_choice/get_head/pyspec_tests/voting_source_within_two_epo OK ++ ForkChoice - minimal/altair/fork_choice/get_proposer_head/pyspec_tests/basic_is_head_root OK ++ ForkChoice - minimal/altair/fork_choice/get_proposer_head/pyspec_tests/basic_is_parent_roo OK + ForkChoice - minimal/altair/fork_choice/on_block/pyspec_tests/basic OK + ForkChoice - minimal/altair/fork_choice/on_block/pyspec_tests/incompatible_justification_u OK + ForkChoice - minimal/altair/fork_choice/on_block/pyspec_tests/incompatible_justification_u OK @@ -3023,6 +3029,8 @@ OK: 45/45 Fail: 0/45 Skip: 0/45 + ForkChoice - minimal/bellatrix/fork_choice/get_head/pyspec_tests/split_tie_breaker_no_atte OK + ForkChoice - minimal/bellatrix/fork_choice/get_head/pyspec_tests/voting_source_beyond_two_ OK + ForkChoice - minimal/bellatrix/fork_choice/get_head/pyspec_tests/voting_source_within_two_ OK ++ ForkChoice - minimal/bellatrix/fork_choice/get_proposer_head/pyspec_tests/basic_is_head_ro OK ++ ForkChoice - minimal/bellatrix/fork_choice/get_proposer_head/pyspec_tests/basic_is_parent_ OK + ForkChoice - minimal/bellatrix/fork_choice/on_block/pyspec_tests/basic OK + ForkChoice - minimal/bellatrix/fork_choice/on_block/pyspec_tests/incompatible_justificatio OK + ForkChoice - minimal/bellatrix/fork_choice/on_block/pyspec_tests/incompatible_justificatio OK @@ -3058,6 +3066,8 @@ OK: 45/45 Fail: 0/45 Skip: 0/45 + ForkChoice - minimal/bellatrix/fork_choice/reorg/pyspec_tests/simple_attempted_reorg_delay OK + ForkChoice - minimal/bellatrix/fork_choice/reorg/pyspec_tests/simple_attempted_reorg_delay OK + ForkChoice - minimal/bellatrix/fork_choice/reorg/pyspec_tests/simple_attempted_reorg_witho OK ++ ForkChoice - minimal/bellatrix/fork_choice/should_override_forkchoice_update/pyspec_tests/ OK ++ ForkChoice - minimal/bellatrix/fork_choice/should_override_forkchoice_update/pyspec_tests/ OK + ForkChoice - minimal/bellatrix/fork_choice/withholding/pyspec_tests/withholding_attack OK + ForkChoice - minimal/bellatrix/fork_choice/withholding/pyspec_tests/withholding_attack_unv OK + ForkChoice - minimal/capella/fork_choice/ex_ante/pyspec_tests/ex_ante_sandwich_with_honest OK @@ -3073,6 +3083,8 @@ OK: 45/45 Fail: 0/45 Skip: 0/45 + ForkChoice - minimal/capella/fork_choice/get_head/pyspec_tests/split_tie_breaker_no_attest OK + ForkChoice - minimal/capella/fork_choice/get_head/pyspec_tests/voting_source_beyond_two_ep OK + ForkChoice - minimal/capella/fork_choice/get_head/pyspec_tests/voting_source_within_two_ep OK ++ ForkChoice - minimal/capella/fork_choice/get_proposer_head/pyspec_tests/basic_is_head_root OK ++ ForkChoice - minimal/capella/fork_choice/get_proposer_head/pyspec_tests/basic_is_parent_ro OK + ForkChoice - minimal/capella/fork_choice/on_block/pyspec_tests/basic OK + ForkChoice - minimal/capella/fork_choice/on_block/pyspec_tests/incompatible_justification_ OK + ForkChoice - minimal/capella/fork_choice/on_block/pyspec_tests/incompatible_justification_ OK @@ -3104,6 +3116,8 @@ OK: 45/45 Fail: 0/45 Skip: 0/45 + ForkChoice - minimal/capella/fork_choice/reorg/pyspec_tests/simple_attempted_reorg_delayed OK + ForkChoice - minimal/capella/fork_choice/reorg/pyspec_tests/simple_attempted_reorg_delayed OK + ForkChoice - minimal/capella/fork_choice/reorg/pyspec_tests/simple_attempted_reorg_without OK ++ ForkChoice - minimal/capella/fork_choice/should_override_forkchoice_update/pyspec_tests/sh OK ++ ForkChoice - minimal/capella/fork_choice/should_override_forkchoice_update/pyspec_tests/sh OK + ForkChoice - minimal/capella/fork_choice/withholding/pyspec_tests/withholding_attack OK + ForkChoice - minimal/capella/fork_choice/withholding/pyspec_tests/withholding_attack_unvia OK + ForkChoice - minimal/deneb/fork_choice/ex_ante/pyspec_tests/ex_ante_sandwich_with_honest_a OK @@ -3119,6 +3133,8 @@ OK: 45/45 Fail: 0/45 Skip: 0/45 + ForkChoice - minimal/deneb/fork_choice/get_head/pyspec_tests/split_tie_breaker_no_attestat OK + ForkChoice - minimal/deneb/fork_choice/get_head/pyspec_tests/voting_source_beyond_two_epoc OK + ForkChoice - minimal/deneb/fork_choice/get_head/pyspec_tests/voting_source_within_two_epoc OK ++ ForkChoice - minimal/deneb/fork_choice/get_proposer_head/pyspec_tests/basic_is_head_root OK ++ ForkChoice - minimal/deneb/fork_choice/get_proposer_head/pyspec_tests/basic_is_parent_root OK + ForkChoice - minimal/deneb/fork_choice/on_block/pyspec_tests/basic OK + ForkChoice - minimal/deneb/fork_choice/on_block/pyspec_tests/incompatible_justification_up OK + ForkChoice - minimal/deneb/fork_choice/on_block/pyspec_tests/incompatible_justification_up OK @@ -3155,10 +3171,12 @@ OK: 45/45 Fail: 0/45 Skip: 0/45 + ForkChoice - minimal/deneb/fork_choice/reorg/pyspec_tests/simple_attempted_reorg_delayed_j OK + ForkChoice - minimal/deneb/fork_choice/reorg/pyspec_tests/simple_attempted_reorg_delayed_j OK + ForkChoice - minimal/deneb/fork_choice/reorg/pyspec_tests/simple_attempted_reorg_without_e OK ++ ForkChoice - minimal/deneb/fork_choice/should_override_forkchoice_update/pyspec_tests/shou OK ++ ForkChoice - minimal/deneb/fork_choice/should_override_forkchoice_update/pyspec_tests/shou OK + ForkChoice - minimal/deneb/fork_choice/withholding/pyspec_tests/withholding_attack OK + ForkChoice - minimal/deneb/fork_choice/withholding/pyspec_tests/withholding_attack_unviabl OK ``` -OK: 185/193 Fail: 0/193 Skip: 8/193 +OK: 199/207 Fail: 0/207 Skip: 8/207 ## Sync ```diff + Sync - minimal/bellatrix/sync/optimistic/pyspec_tests/from_syncing_to_invalid OK @@ -3168,4 +3186,4 @@ OK: 185/193 Fail: 0/193 Skip: 8/193 OK: 3/3 Fail: 0/3 Skip: 0/3 ---TOTAL--- -OK: 2578/2586 Fail: 0/2586 Skip: 8/2586 +OK: 2592/2600 Fail: 0/2600 Skip: 8/2600 diff --git a/beacon_chain/beacon_chain_db.nim b/beacon_chain/beacon_chain_db.nim index 2290505f51..88d96789d0 100644 --- a/beacon_chain/beacon_chain_db.nim +++ b/beacon_chain/beacon_chain_db.nim @@ -793,7 +793,8 @@ proc putBlock*( proc putBlobSidecar*( db: BeaconChainDB, value: BlobSidecar) = - db.blobs.putSZSSZ(blobkey(value.block_root, value.index), value) + let block_root = hash_tree_root(value.signed_block_header.message) + db.blobs.putSZSSZ(blobkey(block_root, value.index), value) proc delBlobSidecar*( db: BeaconChainDB, diff --git a/beacon_chain/consensus_object_pools/blob_quarantine.nim b/beacon_chain/consensus_object_pools/blob_quarantine.nim index c7a78b4048..bc1fb9d8ab 100644 --- a/beacon_chain/consensus_object_pools/blob_quarantine.nim +++ b/beacon_chain/consensus_object_pools/blob_quarantine.nim @@ -7,7 +7,7 @@ {.push raises: [].} -import ../spec/datatypes/deneb +import ../spec/helpers from std/sequtils import mapIt from std/strutils import join @@ -37,8 +37,9 @@ func put*(quarantine: var BlobQuarantine, blobSidecar: ref BlobSidecar) = oldest_blob_key = k break quarantine.blobs.del oldest_blob_key - discard quarantine.blobs.hasKeyOrPut((blobSidecar.block_root, - blobSidecar.index), blobSidecar) + let block_root = hash_tree_root(blobSidecar.signed_block_header.message) + discard quarantine.blobs.hasKeyOrPut( + (block_root, blobSidecar.index), blobSidecar) func blobIndices*(quarantine: BlobQuarantine, digest: Eth2Digest): seq[BlobIndex] = @@ -49,7 +50,21 @@ func blobIndices*(quarantine: BlobQuarantine, digest: Eth2Digest): r func hasBlob*(quarantine: BlobQuarantine, blobSidecar: BlobSidecar): bool = - quarantine.blobs.hasKey((blobSidecar.block_root, blobSidecar.index)) + let block_root = hash_tree_root(blobSidecar.signed_block_header.message) + quarantine.blobs.hasKey((block_root, blobSidecar.index)) + +func hasBlob*( + quarantine: BlobQuarantine, + slot: Slot, + proposer_index: uint64, + index: BlobIndex): bool = + for blob_sidecar in quarantine.blobs.values: + template block_header: untyped = blob_sidecar.signed_block_header.message + if block_header.slot == slot and + block_header.proposer_index == proposer_index and + blob_sidecar.index == index: + return true + false func popBlobs*(quarantine: var BlobQuarantine, digest: Eth2Digest): seq[ref BlobSidecar] = diff --git a/beacon_chain/gossip_processing/eth2_processor.nim b/beacon_chain/gossip_processing/eth2_processor.nim index 71f22f66d0..f9a0acf404 100644 --- a/beacon_chain/gossip_processing/eth2_processor.nim +++ b/beacon_chain/gossip_processing/eth2_processor.nim @@ -245,7 +245,7 @@ proc processSignedBeaconBlock*( Opt.some(self.blobQuarantine[].popBlobs(signedBlock.root)) else: if not self.quarantine[].addBlobless(self.dag.finalizedHead.slot, - signedBlock): + signedBlock): notice "Block quarantine full (blobless)", blockRoot = shortLog(signedBlock.root), blck = shortLog(signedBlock.message), @@ -273,30 +273,26 @@ proc processSignedBeaconBlock*( v -proc processSignedBlobSidecar*( +proc processBlobSidecar*( self: var Eth2Processor, src: MsgSource, - signedBlobSidecar: deneb.SignedBlobSidecar, subnet_id: BlobId): ValidationRes = + blobSidecar: deneb.BlobSidecar, subnet_id: BlobId): ValidationRes = + template block_header: untyped = blobSidecar.signed_block_header.message + let wallTime = self.getCurrentBeaconTime() (afterGenesis, wallSlot) = wallTime.toSlot() logScope: - blob = shortLog(signedBlobSidecar.message) - signature = shortLog(signedBlobSidecar.signature) + blob = shortLog(blobSidecar) wallSlot # Potential under/overflows are fine; would just create odd metrics and logs - let delay = wallTime - signedBlobSidecar.message.slot.start_beacon_time - - if self.blobQuarantine[].hasBlob(signedBlobSidecar.message): - debug "Blob received, already in quarantine", delay - return ValidationRes.ok - else: - debug "Blob received", delay + let delay = wallTime - block_header.slot.start_beacon_time + debug "Blob received", delay let v = self.dag.validateBlobSidecar(self.quarantine, self.blobQuarantine, - signedBlobSidecar, wallTime, subnet_id) + blobSidecar, wallTime, subnet_id) if v.isErr(): debug "Dropping blob", error = v.error() @@ -304,21 +300,19 @@ proc processSignedBlobSidecar*( return v debug "Blob validated, putting in blob quarantine" - self.blobQuarantine[].put(newClone(signedBlobSidecar.message)) + self.blobQuarantine[].put(newClone(blobSidecar)) var skippedBlocks = false - if (let o = self.quarantine[].popBlobless( - signedBlobSidecar.message.block_root); o.isSome): + let block_root = hash_tree_root(block_header) + if (let o = self.quarantine[].popBlobless(block_root); o.isSome): let blobless = o.unsafeGet() if self.blobQuarantine[].hasBlobs(blobless): self.blockProcessor[].enqueueBlock( MsgSource.gossip, ForkedSignedBeaconBlock.init(blobless), - Opt.some(self.blobQuarantine[].popBlobs( - signedBlobSidecar.message.block_root)) - ) + Opt.some(self.blobQuarantine[].popBlobs(block_root))) else: discard self.quarantine[].addBlobless(self.dag.finalizedHead.slot, blobless) diff --git a/beacon_chain/gossip_processing/gossip_validation.nim b/beacon_chain/gossip_processing/gossip_validation.nim index 8ec4538f0e..1f65a83b62 100644 --- a/beacon_chain/gossip_processing/gossip_validation.nim +++ b/beacon_chain/gossip_processing/gossip_validation.nim @@ -182,6 +182,20 @@ func check_attestation_subnet( ok() +# https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.4/specs/deneb/p2p-interface.md#verify_blob_sidecar_inclusion_proof +func verify_blob_sidecar_inclusion_proof( + blob_sidecar: deneb.BlobSidecar): Result[void, ValidationError] = + let gindex = kzg_commitment_inclusion_proof_gindex(blob_sidecar.index) + if not is_valid_merkle_branch( + hash_tree_root(blob_sidecar.kzg_commitment), + blob_sidecar.kzg_commitment_inclusion_proof, + KZG_COMMITMENT_INCLUSION_PROOF_DEPTH, + get_subtree_index(gindex), + blob_sidecar.signed_block_header.message.body_root): + return errReject("Sidecar's inclusion proof not valid") + + ok() + # Gossip Validation # ---------------------------------------------------------------- @@ -302,85 +316,132 @@ template validateBeaconBlockBellatrix( # cannot occur here, because Nimbus's optimistic sync waits for either # `ACCEPTED` or `SYNCING` from the EL to get this far. -# https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.2/specs/deneb/p2p-interface.md#blob_sidecar_subnet_id +# https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.4/specs/deneb/p2p-interface.md#blob_sidecar_subnet_id proc validateBlobSidecar*( dag: ChainDAGRef, quarantine: ref Quarantine, - blobQuarantine: ref BlobQuarantine, sbs: SignedBlobSidecar, + blobQuarantine: ref BlobQuarantine, blob_sidecar: BlobSidecar, wallTime: BeaconTime, subnet_id: BlobId): Result[void, ValidationError] = + # Some of the checks below have been reordered compared to the spec, to + # perform the cheap checks first - in particular, we want to avoid loading + # an `EpochRef` and checking signatures. This reordering might lead to + # different IGNORE/REJECT results in turn affecting gossip scores. + template block_header: untyped = blob_sidecar.signed_block_header.message - # [REJECT] The sidecar is for the correct topic -- - # i.e. sidecar.index matches the topic {index}. - if sbs.message.index != subnet_id: - return dag.checkedReject("SignedBlobSidecar: mismatched gossip topic index") + # [REJECT] The sidecar's index is consistent with `MAX_BLOBS_PER_BLOCK` + # -- i.e. `blob_sidecar.index < MAX_BLOBS_PER_BLOCK` + if not (blob_sidecar.index < MAX_BLOBS_PER_BLOCK): + return dag.checkedReject("BlobSidecar: index inconsistent") - if dag.getBlockRef(sbs.message.block_root).isSome(): - return errIgnore("SignedBlobSidecar: already have block") + # [REJECT] The sidecar is for the correct subnet -- i.e. + # `compute_subnet_for_blob_sidecar(blob_sidecar.index) == subnet_id`. + if not (compute_subnet_for_blob_sidecar(blob_sidecar.index) == subnet_id): + return dag.checkedReject("BlobSidecar: subnet incorrect") # [IGNORE] The sidecar is not from a future slot (with a - # MAXIMUM_GOSSIP_CLOCK_DISPARITY allowance) -- i.e. validate that - # sidecar.slot <= current_slot (a client MAY queue future sidecars + # `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. validate that + # `block_header.slot <= current_slot` (a client MAY queue future sidecars # for processing at the appropriate slot). - if not (sbs.message.slot <= + if not (block_header.slot <= (wallTime + MAXIMUM_GOSSIP_CLOCK_DISPARITY).slotOrZero): - return errIgnore("SignedBlobSidecar: slot too high") - - # [IGNORE] The block is from a slot greater than the latest - # finalized slot -- i.e. validate that - # signed_beacon_block.message.slot > - # compute_start_slot_at_epoch(state.finalized_checkpoint.epoch) - if not (sbs.message.slot > dag.finalizedHead.slot): - return errIgnore("SignedBlobSidecar: slot already finalized") - - # [IGNORE] The block's parent (defined by block.parent_root) has - # been seen (via both gossip and non-gossip sources) (a client MAY - # queue blocks for processing once the parent block is retrieved). - # [REJECT] The sidecar's block's parent (defined by sidecar.block_parent_root) - # passes validation. - let parentRes = dag.getBlockRef(sbs.message.block_parent_root) - if parentRes.isErr: - if sbs.message.block_parent_root in quarantine[].unviable: - return dag.checkedReject("SignedBlobSidecar: parent not validated") + return errIgnore("BlobSidecar: slot too high") + + # [IGNORE] The sidecar is from a slot greater than the latest + # finalized slot -- i.e. validate that `block_header.slot > + # compute_start_slot_at_epoch(state.finalized_checkpoint.epoch)` + if not (block_header.slot > dag.finalizedHead.slot): + return errIgnore("BlobSidecar: slot already finalized") + + # [IGNORE] The sidecar is the first sidecar for the tuple + # (block_header.slot, block_header.proposer_index, blob_sidecar.index) + # with valid header signature, sidecar inclusion proof, and kzg proof. + let block_root = hash_tree_root(block_header) + if dag.getBlockRef(block_root).isSome(): + return errIgnore("BlobSidecar: already have block") + if blobQuarantine[].hasBlob( + block_header.slot, block_header.proposer_index, blob_sidecar.index): + return errIgnore("BlobSidecar: already have valid blob from same proposer") + + # [REJECT] The sidecar's inclusion proof is valid as verified by + # `verify_blob_sidecar_inclusion_proof(blob_sidecar)`. + block: + let v = verify_blob_sidecar_inclusion_proof(blob_sidecar) + if v.isErr: + return dag.checkedReject(v.error) + + # [IGNORE] The sidecar's block's parent (defined by + # `block_header.parent_root`) has been seen (via both gossip and + # non-gossip sources) (a client MAY queue sidecars for processing + # once the parent block is retrieved). + # + # [REJECT] The sidecar's block's parent (defined by + # `block_header.parent_root`) passes validation. + let parent = dag.getBlockRef(block_header.parent_root).valueOr: + if block_header.parent_root in quarantine[].unviable: + quarantine[].addUnviable(block_root) + return dag.checkedReject("BlobSidecar: parent not validated") else: - return errIgnore("SignedBlobSidecar: parent not found") - template parent: untyped = parentRes.get + quarantine[].addMissing(block_header.parent_root) + return errIgnore("BlobSidecar: parent not found") # [REJECT] The sidecar is from a higher slot than the sidecar's - # block's parent (defined by sidecar.block_parent_root). - if sbs.message.slot <= parent.bid.slot: - return dag.checkedReject("SignedBlobSidecar: slot lower than parents'") + # block's parent (defined by `block_header.parent_root`). + if not (block_header.slot > parent.bid.slot): + return dag.checkedReject("BlobSidecar: slot lower than parents'") - # [REJECT] The sidecar is proposed by the expected proposer_index + # [REJECT] The current finalized_checkpoint is an ancestor of the sidecar's + # block -- i.e. `get_checkpoint_block(store, block_header.parent_root, + # store.finalized_checkpoint.epoch) == store.finalized_checkpoint.root`. + let + finalized_checkpoint = getStateField(dag.headState, finalized_checkpoint) + ancestor = get_ancestor(parent, finalized_checkpoint.epoch.start_slot) + + if ancestor.isNil: + # This shouldn't happen: we should always be able to trace the parent back + # to the finalized checkpoint (else it wouldn't be in the DAG) + return errIgnore("BlobSidecar: Can't find ancestor") + + if not ( + finalized_checkpoint.root == ancestor.root or + finalized_checkpoint.root.isZero): + quarantine[].addUnviable(block_root) + return dag.checkedReject( + "BlobSidecar: Finalized checkpoint not an ancestor") + + # [REJECT] The sidecar is proposed by the expected `proposer_index` # for the block's slot in the context of the current shuffling - # (defined by block_parent_root/slot). If the proposer_index - # cannot immediately be verified against the expected shuffling, - # the sidecar MAY be queued for later processing while proposers + # (defined by `block_header.parent_root`/`block_header.slot`). + # If the proposer_index cannot immediately be verified against the expected + # shuffling, the sidecar MAY be queued for later processing while proposers # for the block's branch are calculated -- in such a case do not # REJECT, instead IGNORE this message. - let - proposer = getProposer( - dag, parent, sbs.message.slot).valueOr: - warn "cannot compute proposer for blob" - return errIgnore("SignedBlobSidecar: Cannot compute proposer") - - if uint64(proposer) != sbs.message.proposer_index: - return dag.checkedReject("SignedBlobSidecar: Unexpected proposer") - - # [REJECT] The proposer signature, signed_blob_sidecar.signature, - # is valid as verified by verify_sidecar_signature. - if not verify_blob_signature( - dag.forkAtEpoch(sbs.message.slot.epoch), - getStateField(dag.headState, genesis_validators_root), - sbs.message.slot, - sbs.message, - dag.validatorKey(proposer).get(), - sbs.signature): - return dag.checkedReject("SignedBlobSidecar: invalid blob signature") - - # [IGNORE] The sidecar is the only sidecar with valid signature - # received for the tuple (sidecar.block_root, sidecar.index). - if blobQuarantine[].hasBlob(sbs.message): - return errIgnore( - "SignedBlobSidecar: already have blob with valid signature") + let proposer = getProposer(dag, parent, block_header.slot).valueOr: + warn "cannot compute proposer for blob" + return errIgnore("BlobSidecar: Cannot compute proposer") # internal issue + + if uint64(proposer) != block_header.proposer_index: + return dag.checkedReject("BlobSidecar: Unexpected proposer") + + # [REJECT] The proposer signature of `blob_sidecar.signed_block_header`, + # is valid with respect to the `block_header.proposer_index` pubkey. + if not verify_block_signature( + dag.forkAtEpoch(block_header.slot.epoch), + getStateField(dag.headState, genesis_validators_root), + block_header.slot, + block_root, + dag.validatorKey(proposer).get(), + blob_sidecar.signed_block_header.signature): + return dag.checkedReject("BlobSidecar: Invalid proposer signature") + + # [REJECT] The sidecar's blob is valid as verified by `verify_blob_kzg_proof( + # blob_sidecar.blob, blob_sidecar.kzg_commitment, blob_sidecar.kzg_proof)`. + block: + let ok = verifyProof( + blob_sidecar.blob, + blob_sidecar.kzg_commitment, + blob_sidecar.kzg_proof).valueOr: + return dag.checkedReject("BlobSidecar: blob verify failed") + if not ok: + return dag.checkedReject("BlobSidecar: blob invalid") ok() @@ -498,7 +559,7 @@ proc validateBeaconBlock*( blockRoot = shortLog(signed_beacon_block.root), blck = shortLog(signed_beacon_block.message), signature = shortLog(signed_beacon_block.signature) - return errIgnore("BeaconBlock: Parent not found") + return errIgnore("BeaconBlock: parent not found") # Continues block parent validity checking in optimistic case, where it does # appear as a `BlockRef` (and not handled above) but isn't usable for gossip @@ -539,12 +600,12 @@ proc validateBeaconBlock*( let proposer = getProposer( dag, parent, signed_beacon_block.message.slot).valueOr: - warn "cannot compute proposer for message" + warn "cannot compute proposer for block" return errIgnore("BeaconBlock: Cannot compute proposer") # internal issue if uint64(proposer) != signed_beacon_block.message.proposer_index: quarantine[].addUnviable(signed_beacon_block.root) - return dag.checkedReject("BeaconBlock: Unexpected proposer proposer") + return dag.checkedReject("BeaconBlock: Unexpected proposer") # [REJECT] The proposer signature, signed_beacon_block.signature, is valid # with respect to the proposer_index pubkey. diff --git a/beacon_chain/networking/eth2_network.nim b/beacon_chain/networking/eth2_network.nim index 23f6c5ad82..25049c8a3d 100644 --- a/beacon_chain/networking/eth2_network.nim +++ b/beacon_chain/networking/eth2_network.nim @@ -2678,7 +2678,7 @@ proc broadcastBeaconBlock*( node.broadcast(topic, blck) proc broadcastBlobSidecar*( - node: Eth2Node, subnet_id: BlobId, blob: deneb.SignedBlobSidecar): + node: Eth2Node, subnet_id: BlobId, blob: deneb.BlobSidecar): Future[SendResult] = let forkPrefix = node.forkDigestAtEpoch(node.getWallEpoch) diff --git a/beacon_chain/nimbus_beacon_node.nim b/beacon_chain/nimbus_beacon_node.nim index 39c91cd110..ac90b41f69 100644 --- a/beacon_chain/nimbus_beacon_node.nim +++ b/beacon_chain/nimbus_beacon_node.nim @@ -1734,17 +1734,17 @@ proc installMessageValidators(node: BeaconNode) = when consensusFork >= ConsensusFork.Deneb: # blob_sidecar_{subnet_id} - # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.2/specs/deneb/p2p-interface.md#blob_sidecar_subnet_id + # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.4/specs/deneb/p2p-interface.md#blob_sidecar_subnet_id for it in BlobId: closureScope: # Needed for inner `proc`; don't lift it out of loop. let subnet_id = it node.network.addValidator( getBlobSidecarTopic(digest, subnet_id), proc ( - signedBlobSidecar: SignedBlobSidecar + blobSidecar: deneb.BlobSidecar ): ValidationResult = toValidationResult( - node.processor[].processSignedBlobSidecar( - MsgSource.gossip, signedBlobSidecar, subnet_id))) + node.processor[].processBlobSidecar( + MsgSource.gossip, blobSidecar, subnet_id))) node.installLightClientMessageValidators() diff --git a/beacon_chain/rpc/rest_beacon_api.nim b/beacon_chain/rpc/rest_beacon_api.nim index 56b741999b..944002f511 100644 --- a/beacon_chain/rpc/rest_beacon_api.nim +++ b/beacon_chain/rpc/rest_beacon_api.nim @@ -873,28 +873,29 @@ proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) = of ConsensusFork.Phase0: var blck = restBlock.phase0Data blck.root = hash_tree_root(blck.message) - await node.router.routeSignedBeaconBlock(blck, - Opt.none(SignedBlobSidecars)) + await node.router.routeSignedBeaconBlock( + blck, Opt.none(seq[BlobSidecar])) of ConsensusFork.Altair: var blck = restBlock.altairData blck.root = hash_tree_root(blck.message) - await node.router.routeSignedBeaconBlock(blck, - Opt.none(SignedBlobSidecars)) + await node.router.routeSignedBeaconBlock( + blck, Opt.none(seq[BlobSidecar])) of ConsensusFork.Bellatrix: var blck = restBlock.bellatrixData blck.root = hash_tree_root(blck.message) - await node.router.routeSignedBeaconBlock(blck, - Opt.none(SignedBlobSidecars)) + await node.router.routeSignedBeaconBlock( + blck, Opt.none(seq[BlobSidecar])) of ConsensusFork.Capella: var blck = restBlock.capellaData blck.root = hash_tree_root(blck.message) - await node.router.routeSignedBeaconBlock(blck, - Opt.none(SignedBlobSidecars)) + await node.router.routeSignedBeaconBlock( + blck, Opt.none(seq[BlobSidecar])) of ConsensusFork.Deneb: var blck = restBlock.denebData.signed_block blck.root = hash_tree_root(blck.message) await node.router.routeSignedBeaconBlock( - blck, Opt.some(asSeq restBlock.denebData.signed_blob_sidecars)) + blck, Opt.some(blck.create_blob_sidecars( + restBlock.denebData.kzg_proofs, restBlock.denebData.blobs))) if res.isErr(): return RestApiResponse.jsonError( @@ -948,28 +949,29 @@ proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) = of ConsensusFork.Phase0: var blck = restBlock.phase0Data blck.root = hash_tree_root(blck.message) - await node.router.routeSignedBeaconBlock(blck, - Opt.none(SignedBlobSidecars)) + await node.router.routeSignedBeaconBlock( + blck, Opt.none(seq[BlobSidecar])) of ConsensusFork.Altair: var blck = restBlock.altairData blck.root = hash_tree_root(blck.message) - await node.router.routeSignedBeaconBlock(blck, - Opt.none(SignedBlobSidecars)) + await node.router.routeSignedBeaconBlock( + blck, Opt.none(seq[BlobSidecar])) of ConsensusFork.Bellatrix: var blck = restBlock.bellatrixData blck.root = hash_tree_root(blck.message) - await node.router.routeSignedBeaconBlock(blck, - Opt.none(SignedBlobSidecars)) + await node.router.routeSignedBeaconBlock( + blck, Opt.none(seq[BlobSidecar])) of ConsensusFork.Capella: var blck = restBlock.capellaData blck.root = hash_tree_root(blck.message) - await node.router.routeSignedBeaconBlock(blck, - Opt.none(SignedBlobSidecars)) + await node.router.routeSignedBeaconBlock( + blck, Opt.none(seq[BlobSidecar])) of ConsensusFork.Deneb: var blck = restBlock.denebData.signed_block blck.root = hash_tree_root(blck.message) await node.router.routeSignedBeaconBlock( - blck, Opt.some(asSeq restBlock.denebData.signed_blob_sidecars)) + blck, Opt.some(blck.create_blob_sidecars( + restBlock.denebData.kzg_proofs, restBlock.denebData.blobs))) if res.isErr(): return RestApiResponse.jsonError( @@ -1066,8 +1068,8 @@ proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) = let res = withBlck(forked): forkyBlck.root = hash_tree_root(forkyBlck.message) - await node.router.routeSignedBeaconBlock(forkyBlck, - Opt.none(SignedBlobSidecars)) + await node.router.routeSignedBeaconBlock( + forkyBlck, Opt.none(seq[BlobSidecar])) if res.isErr(): return RestApiResponse.jsonError( diff --git a/beacon_chain/rpc/rest_validator_api.nim b/beacon_chain/rpc/rest_validator_api.nim index c016f899f7..734d974d51 100644 --- a/beacon_chain/rpc/rest_validator_api.nim +++ b/beacon_chain/rpc/rest_validator_api.nim @@ -407,26 +407,11 @@ proc installValidatorApiHandlers*(router: var RestRouter, node: BeaconNode) = withBlck(message.blck): let data = when consensusFork >= ConsensusFork.Deneb: - let bundle = message.blobsBundleOpt.get() - let blockRoot = hash_tree_root(forkyBlck) - var sidecars = newSeqOfCap[BlobSidecar](bundle.blobs.len) - for i in 0..= ConsensusFork.Deneb: + if kzg_proofs.isNone(): + reader.raiseUnexpectedValue("Field `kzg_proofs` is missing") + if blobs.isNone(): + reader.raiseUnexpectedValue("Field `blobs` is missing") + else: + if kzg_proofs.isSome(): + reader.raiseUnexpectedValue("Field `kzg_proofs` found but unsupported") + if blobs.isSome(): + reader.raiseUnexpectedValue("Field `blobs` found but unsupported") + case blck.kind of ConsensusFork.Phase0: value = RestPublishedSignedBlockContents( @@ -1865,7 +1885,8 @@ proc readValue*(reader: var JsonReader[RestJson], denebData: DenebSignedBlockContents( # Constructed to be internally consistent signed_block: signed_message.get().distinctBase.denebData, - signed_blob_sidecars: signed_blob_sidecars.get() + kzg_proofs: kzg_proofs.get(), + blobs: blobs.get() ) ) diff --git a/beacon_chain/spec/eth2_apis/rest_types.nim b/beacon_chain/spec/eth2_apis/rest_types.nim index c36d25a610..9280cb6668 100644 --- a/beacon_chain/spec/eth2_apis/rest_types.nim +++ b/beacon_chain/spec/eth2_apis/rest_types.nim @@ -317,7 +317,8 @@ type DenebSignedBlockContents* = object signed_block*: deneb.SignedBeaconBlock - signed_blob_sidecars*: List[SignedBlobSidecar, Limit MAX_BLOBS_PER_BLOCK] + kzg_proofs*: deneb.KzgProofs + blobs*: deneb.Blobs RestPublishedSignedBlockContents* = object case kind*: ConsensusFork @@ -339,7 +340,8 @@ type DenebBlockContents* = object `block`*: deneb.BeaconBlock - blob_sidecars*: List[BlobSidecar, Limit MAX_BLOBS_PER_BLOCK] + kzg_proofs*: deneb.KzgProofs + blobs*: deneb.Blobs ProduceBlockResponseV2* = object case kind*: ConsensusFork diff --git a/beacon_chain/spec/forks.nim b/beacon_chain/spec/forks.nim index 1360cd2f0e..8dc3d39ad8 100644 --- a/beacon_chain/spec/forks.nim +++ b/beacon_chain/spec/forks.nim @@ -905,8 +905,9 @@ template withStateAndBlck*( func toBeaconBlockHeader*( blck: SomeForkyBeaconBlock | - capella_mev.BlindedBeaconBlock | deneb_mev.BlindedBeaconBlock): - BeaconBlockHeader = + capella_mev.BlindedBeaconBlock | + deneb_mev.BlindedBeaconBlock +): BeaconBlockHeader = ## Reduce a given `BeaconBlock` to its `BeaconBlockHeader`. BeaconBlockHeader( slot: blck.slot, @@ -918,7 +919,7 @@ func toBeaconBlockHeader*( template toBeaconBlockHeader*( blck: SomeForkySignedBeaconBlock): BeaconBlockHeader = ## Reduce a given `SignedBeaconBlock` to its `BeaconBlockHeader`. - blck.message.toBeaconBlockHeader + blck.message.toBeaconBlockHeader() template toBeaconBlockHeader*( blckParam: ForkedMsgTrustedSignedBeaconBlock | @@ -926,6 +927,16 @@ template toBeaconBlockHeader*( ## Reduce a given signed beacon block to its `BeaconBlockHeader`. withBlck(blckParam): forkyBlck.toBeaconBlockHeader() +func toSignedBeaconBlockHeader*( + signedBlock: SomeForkySignedBeaconBlock | + capella_mev.SignedBlindedBeaconBlock | + deneb_mev.SignedBlindedBeaconBlock +): SignedBeaconBlockHeader = + ## Reduce a given `SignedBeaconBlock` to its `SignedBeaconBlockHeader`. + SignedBeaconBlockHeader( + message: signedBlock.message.toBeaconBlockHeader(), + signature: signedBlock.signature) + func genesisFork*(cfg: RuntimeConfig): Fork = Fork( previous_version: cfg.GENESIS_FORK_VERSION, diff --git a/beacon_chain/spec/helpers.nim b/beacon_chain/spec/helpers.nim index b392d612ea..dc6c448466 100644 --- a/beacon_chain/spec/helpers.nim +++ b/beacon_chain/spec/helpers.nim @@ -211,6 +211,54 @@ func has_flag*(flags: ParticipationFlags, flag_index: TimelyFlag): bool = let flag = ParticipationFlags(1'u8 shl ord(flag_index)) (flags and flag) == flag +func create_blob_sidecars*( + forkyBlck: deneb.SignedBeaconBlock, + kzg_proofs: KzgProofs, + blobs: Blobs): seq[BlobSidecar] = + template kzg_commitments: untyped = + forkyBlck.message.body.blob_kzg_commitments + doAssert kzg_proofs.len == blobs.len + doAssert kzg_proofs.len == kzg_commitments.len + + var res = newSeqOfCap[BlobSidecar](blobs.len) + let signedBlockHeader = forkyBlck.toSignedBeaconBlockHeader() + for i in 0 ..< blobs.lenu64: + var sidecar = BlobSidecar( + index: i, + blob: blobs[i], + kzg_commitment: kzg_commitments[i], + kzg_proof: kzg_proofs[i], + signed_block_header: signedBlockHeader) + forkyBlck.message.body.build_proof( + kzg_commitment_inclusion_proof_gindex(i), + sidecar.kzg_commitment_inclusion_proof).expect("Valid gindex") + res.add(sidecar) + res + +func create_blob_sidecars*( + forkyBlck: deneb_mev.SignedBlindedBeaconBlock, + kzg_proofs: KzgProofs, + blob_roots: BlobRoots): seq[BlindedBlobSidecar] = + template kzg_commitments: untyped = + forkyBlck.message.body.blob_kzg_commitments + doAssert kzg_proofs.len == blob_roots.len + doAssert kzg_proofs.len == kzg_commitments.len + + var res = newSeqOfCap[BlindedBlobSidecar](blob_roots.len) + let signedBlockHeader = forkyBlck.toSignedBeaconBlockHeader() + for i in 0 ..< blob_roots.lenu64: + var sidecar = BlindedBlobSidecar( + index: i, + blob_root: blob_roots[i], + kzg_commitment: kzg_commitments[i], + kzg_proof: kzg_proofs[i], + signed_block_header: signedBlockHeader) + forkyBlck.message.body.build_proof( + kzg_commitment_inclusion_proof_gindex(i), + sidecar.kzg_commitment_inclusion_proof).expect("Valid gindex") + res.add(sidecar) + res + # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/altair/light-client/sync-protocol.md#is_sync_committee_update template is_sync_committee_update*(update: SomeForkyLightClientUpdate): bool = when update is SomeForkyLightClientUpdateWithSyncCommittee: diff --git a/beacon_chain/spec/mev/deneb_mev.nim b/beacon_chain/spec/mev/deneb_mev.nim index 736ed0bf73..f64f87860c 100644 --- a/beacon_chain/spec/mev/deneb_mev.nim +++ b/beacon_chain/spec/mev/deneb_mev.nim @@ -69,25 +69,18 @@ type # https://github.com/ethereum/builder-specs/blob/534e4f81276b8346d785ed9aba12c4c74b927ec6/specs/deneb/builder.md#blindedblobsidecar BlindedBlobSidecar* = object - block_root*: Eth2Digest index*: uint64 - slot*: uint64 - block_parent_root*: Eth2Digest - proposer_index*: uint64 blob_root*: Eth2Digest kzg_commitment*: KZGCommitment kzg_proof*: KZGProof - - # https://github.com/ethereum/builder-specs/blob/534e4f81276b8346d785ed9aba12c4c74b927ec6/specs/deneb/builder.md#signedblindedblobsidecar - SignedBlindedBlobSidecar* = object - message*: BlindedBlobSidecar - signature*: ValidatorSig + signed_block_header*: SignedBeaconBlockHeader + kzg_commitment_inclusion_proof*: + array[KZG_COMMITMENT_INCLUSION_PROOF_DEPTH, Eth2Digest] # https://github.com/ethereum/builder-specs/blob/534e4f81276b8346d785ed9aba12c4c74b927ec6/specs/deneb/builder.md#signedblindedblockcontents SignedBlindedBeaconBlockContents* = object signed_blinded_block*: deneb_mev.SignedBlindedBeaconBlock - signed_blinded_blob_sidecars*: - List[SignedBlindedBlobSidecar, Limit MAX_BLOBS_PER_BLOCK] + blinded_blob_sidecars*: List[BlindedBlobSidecar, Limit MAX_BLOBS_PER_BLOCK] # https://github.com/ethereum/builder-specs/blob/534e4f81276b8346d785ed9aba12c4c74b927ec6/specs/deneb/builder.md#executionpayloadandblobsbundle ExecutionPayloadAndBlobsBundle* = object diff --git a/beacon_chain/spec/signatures.nim b/beacon_chain/spec/signatures.nim index 1aa318537e..417c751223 100644 --- a/beacon_chain/spec/signatures.nim +++ b/beacon_chain/spec/signatures.nim @@ -88,15 +88,6 @@ func compute_block_signing_root*( fork, DOMAIN_BEACON_PROPOSER, epoch, genesis_validators_root) compute_signing_root(blck, domain) -func compute_blob_signing_root( - fork: Fork, genesis_validators_root: Eth2Digest, slot: Slot, - blob: BlindedBlobSidecar | BlobSidecar): Eth2Digest = - let - epoch = epoch(slot) - domain = get_domain(fork, DOMAIN_BLOB_SIDECAR, epoch, - genesis_validators_root) - compute_signing_root(blob, domain) - # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/phase0/validator.md#signature func get_block_signature*( fork: Fork, genesis_validators_root: Eth2Digest, slot: Slot, @@ -106,16 +97,6 @@ func get_block_signature*( blsSign(privkey, signing_root.data) -# https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.3/specs/deneb/validator.md#constructing-the-signedblobsidecars -proc get_blob_sidecar_signature*( - fork: Fork, genesis_validators_root: Eth2Digest, slot: Slot, - blob: BlindedBlobSidecar | BlobSidecar, privkey: ValidatorPrivKey): - CookedSig = - let signing_root = compute_blob_signing_root( - fork, genesis_validators_root, slot, blob) - - blsSign(privkey, signing_root.data) - proc verify_block_signature*( fork: Fork, genesis_validators_root: Eth2Digest, slot: Slot, blck: Eth2Digest | SomeForkyBeaconBlock | BeaconBlockHeader, @@ -127,17 +108,6 @@ proc verify_block_signature*( blsVerify(pubkey, signing_root.data, signature) -proc verify_blob_signature*( - fork: Fork, genesis_validators_root: Eth2Digest, slot: Slot, - blobSidecar: BlobSidecar, - pubkey: ValidatorPubKey | CookedPubKey, signature: SomeSig): bool = - withTrust(signature): - let - signing_root = compute_blob_signing_root( - fork, genesis_validators_root, slot, blobSidecar) - - blsVerify(pubkey, signing_root.data, signature) - func compute_aggregate_and_proof_signing_root*( fork: Fork, genesis_validators_root: Eth2Digest, aggregate_and_proof: AggregateAndProof): Eth2Digest = diff --git a/beacon_chain/sync/request_manager.nim b/beacon_chain/sync/request_manager.nim index d37a13d504..045943e44d 100644 --- a/beacon_chain/sync/request_manager.nim +++ b/beacon_chain/sync/request_manager.nim @@ -93,9 +93,10 @@ proc checkResponse(idList: seq[BlobIdentifier], if len(blobs) > len(idList): return false for blob in blobs: + let block_root = hash_tree_root(blob.signed_block_header.message) var found = false for id in idList: - if id.block_root == blob.block_root and + if id.block_root == block_root and id.index == blob.index: found = true break @@ -204,8 +205,9 @@ proc fetchBlobsFromNetwork(self: RequestManager, self.blobQuarantine[].put(b) var curRoot: Eth2Digest for b in ublobs: - if b.block_root != curRoot: - curRoot = b.block_root + let block_root = hash_tree_root(b.signed_block_header.message) + if block_root != curRoot: + curRoot = block_root if (let o = self.quarantine[].popBlobless(curRoot); o.isSome): let b = o.unsafeGet() discard await self.blockVerifier(ForkedSignedBeaconBlock.init(b), false) diff --git a/beacon_chain/sync/sync_manager.nim b/beacon_chain/sync/sync_manager.nim index 2bd07a20ae..c0c35567ee 100644 --- a/beacon_chain/sync/sync_manager.nim +++ b/beacon_chain/sync/sync_manager.nim @@ -247,9 +247,9 @@ func groupBlobs*[T](req: SyncRequest[T], # reached end of blobs, have more blobless blocks break for blob in blobs[blobCursor..len(blobs)-1]: - if blob.slot < slot: + if blob.signed_block_header.message.slot < slot: return Result[seq[BlobSidecars], string].err "invalid blob sequence" - if blob.slot==slot: + if blob.signed_block_header.message.slot == slot: grouped[i].add(blob) blobCursor = blobCursor + 1 i = i + 1 @@ -439,7 +439,7 @@ proc syncStep[A, B](man: SyncManager[A, B], index: int, peer: A) {.async.} = blobs_map = blobSmap, request = req if len(blobData) > 0: - let slots = mapIt(blobData, it[].slot) + let slots = mapIt(blobData, it[].signed_block_header.message.slot) let uniqueSlots = foldl(slots, combine(a, b), @[slots[0]]) if not(checkResponse(req, uniqueSlots)): peer.updateScore(PeerScoreBadResponse) @@ -464,7 +464,8 @@ proc syncStep[A, B](man: SyncManager[A, B], index: int, peer: A) {.async.} = man.queue.push(req) return for i, blk in blockData: - if len(blobs[i]) > 0 and blk[].slot != blobs[i][0].slot: + if len(blobs[i]) > 0 and blk[].slot != + blobs[i][0].signed_block_header.message.slot: peer.updateScore(PeerScoreNoValues) man.queue.push(req) debug "block and blobs data have inconsistent slots" diff --git a/beacon_chain/sync/sync_queue.nim b/beacon_chain/sync/sync_queue.nim index 9e1f306cbe..fb2fbb7628 100644 --- a/beacon_chain/sync/sync_queue.nim +++ b/beacon_chain/sync/sync_queue.nim @@ -119,9 +119,9 @@ proc getShortMap*[T](req: SyncRequest[T], if cur >= lenu64(data): res.add('|') continue - if slot == data[cur].slot: + if slot == data[cur].signed_block_header.message.slot: for k in cur..= lenu64(data) or slot != data[k].slot: + if k >= lenu64(data) or slot != data[k].signed_block_header.message.slot: res.add('|') break else: diff --git a/beacon_chain/validator_client/block_service.nim b/beacon_chain/validator_client/block_service.nim index 995d3c377e..a054647775 100644 --- a/beacon_chain/validator_client/block_service.nim +++ b/beacon_chain/validator_client/block_service.nim @@ -21,12 +21,11 @@ const logScope: service = ServiceName type - BlobList = List[BlobSidecar, Limit MAX_BLOBS_PER_BLOCK] - PreparedBeaconBlock = object blockRoot*: Eth2Digest data*: ForkedBeaconBlock - blobsOpt*: Opt[BlobList] + kzgProofsOpt*: Opt[deneb.KzgProofs] + blobsOpt*: Opt[deneb.Blobs] PreparedBlindedBeaconBlock = object blockRoot*: Eth2Digest @@ -66,27 +65,34 @@ proc produceBlock( let blck = produceBlockResponse.phase0Data return Opt.some(PreparedBeaconBlock(blockRoot: hash_tree_root(blck), data: ForkedBeaconBlock.init(blck), - blobsOpt: Opt.none(BlobList))) + kzgProofsOpt: Opt.none(deneb.KzgProofs), + blobsOpt: Opt.none(deneb.Blobs))) of ConsensusFork.Altair: let blck = produceBlockResponse.altairData return Opt.some(PreparedBeaconBlock(blockRoot: hash_tree_root(blck), data: ForkedBeaconBlock.init(blck), - blobsOpt: Opt.none(BlobList))) + kzgProofsOpt: Opt.none(deneb.KzgProofs), + blobsOpt: Opt.none(deneb.Blobs))) of ConsensusFork.Bellatrix: let blck = produceBlockResponse.bellatrixData return Opt.some(PreparedBeaconBlock(blockRoot: hash_tree_root(blck), data: ForkedBeaconBlock.init(blck), - blobsOpt: Opt.none(BlobList))) + kzgProofsOpt: Opt.none(deneb.KzgProofs), + blobsOpt: Opt.none(deneb.Blobs))) of ConsensusFork.Capella: let blck = produceBlockResponse.capellaData return Opt.some(PreparedBeaconBlock(blockRoot: hash_tree_root(blck), data: ForkedBeaconBlock.init(blck), - blobsOpt: Opt.none(BlobList))) + kzgProofsOpt: Opt.none(deneb.KzgProofs), + blobsOpt: Opt.none(deneb.Blobs))) of ConsensusFork.Deneb: - let blck = produceBlockResponse.denebData.`block` - let blobs = produceBlockResponse.denebData.blob_sidecars + let + blck = produceBlockResponse.denebData.`block` + kzgProofs = produceBlockResponse.denebData.kzg_proofs + blobs = produceBlockResponse.denebData.blobs return Opt.some(PreparedBeaconBlock(blockRoot: hash_tree_root(blck), data: ForkedBeaconBlock.init(blck), + kzgProofsOpt: Opt.some(kzgProofs), blobsOpt: Opt.some(blobs))) proc produceBlindedBlock( @@ -388,30 +394,14 @@ proc publishBlock(vc: ValidatorClientRef, currentSlot, slot: Slot, root: preparedBlock.blockRoot, signature: signature)) of ConsensusFork.Deneb: - let blobs = preparedBlock.blobsOpt.get() - var signed: seq[SignedBlobSidecar] = @[] - for i in 0..= ConsensusFork.Deneb: - var sidecars: seq[BlobSidecar] - let bundle = collectedBids.engineBlockFut.read.get().blobsBundleOpt.get - let (blobs, commitments, proofs) = ( - bundle.blobs, bundle.commitments, bundle.proofs) - for i in 0..= ConsensusFork.Deneb: + template blobsBundle: untyped = + collectedBids.engineBlockFut.read.get.blobsBundleOpt.get + Opt.some(signedBlock.create_blob_sidecars( + blobsBundle.proofs, blobsBundle.blobs)) else: - static: doAssert "Unknown SignedBeaconBlock type" - - newBlockRef = - (await node.router.routeSignedBeaconBlock(signedBlock, signedBlobs)).valueOr: - return head # Errors logged in router + Opt.none(seq[BlobSidecar]) + newBlockRef = ( + await node.router.routeSignedBeaconBlock(signedBlock, blobsOpt) + ).valueOr: + return head # Errors logged in router if newBlockRef.isNone(): return head # Validation errors logged in router diff --git a/beacon_chain/validators/message_router.nim b/beacon_chain/validators/message_router.nim index 431099ff3b..86dc5614df 100644 --- a/beacon_chain/validators/message_router.nim +++ b/beacon_chain/validators/message_router.nim @@ -82,14 +82,10 @@ template blockProcessor(router: MessageRouter): ref BlockProcessor = template getCurrentBeaconTime(router: MessageRouter): BeaconTime = router.processor[].getCurrentBeaconTime() -type SignedBlobSidecars* = seq[SignedBlobSidecar] -func shortLog*(v: SignedBlobSidecars): auto = - "[" & v.mapIt(shortLog(it)).join(", ") & "]" - type RouteBlockResult = Result[Opt[BlockRef], cstring] proc routeSignedBeaconBlock*( router: ref MessageRouter, blck: ForkySignedBeaconBlock, - blobsOpt: Opt[SignedBlobSidecars]): Future[RouteBlockResult] {.async.} = + blobsOpt: Opt[seq[BlobSidecar]]): Future[RouteBlockResult] {.async.} = ## Validate and broadcast beacon block, then add it to the block database ## Returns the new Head when block is added successfully to dag, none when ## block passes validation but is not added, and error otherwise @@ -112,8 +108,8 @@ proc routeSignedBeaconBlock*( let blobs = blobsOpt.get() let kzgCommits = blck.message.body.blob_kzg_commitments.asSeq if blobs.len > 0 or kzgCommits.len > 0: - let res = validate_blobs(kzgCommits, blobs.mapIt(it.message.blob), - blobs.mapIt(it.message.kzg_proof)) + let res = validate_blobs(kzgCommits, blobs.mapIt(it.blob), + blobs.mapIt(it.kzg_proof)) if res.isErr(): warn "blobs failed validation", blockRoot = shortLog(blck.root), @@ -145,26 +141,26 @@ proc routeSignedBeaconBlock*( blockRoot = shortLog(blck.root), blck = shortLog(blck.message), signature = shortLog(blck.signature), error = res.error() - var blobs = Opt.none(seq[ref BlobSidecar]) + var blobRefs = Opt.none(BlobSidecars) if blobsOpt.isSome(): - let signedBlobs = blobsOpt.get() - var workers = newSeq[Future[SendResult]](signedBlobs.len) - for i in 0..