From 809ea98c4413c2f678be9a97fff316f2374d7b2f Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Fri, 29 Nov 2024 22:23:20 +0100 Subject: [PATCH 01/20] Reduce blobs lookup min wait time to 0 (#8864) --- .../statetransition/util/BlockBlobSidecarsTrackersPoolImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/util/BlockBlobSidecarsTrackersPoolImpl.java b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/util/BlockBlobSidecarsTrackersPoolImpl.java index c17529507a6..d305fb55fa8 100644 --- a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/util/BlockBlobSidecarsTrackersPoolImpl.java +++ b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/util/BlockBlobSidecarsTrackersPoolImpl.java @@ -93,7 +93,7 @@ public class BlockBlobSidecarsTrackersPoolImpl extends AbstractIgnoringFutureHis static final String GAUGE_BLOB_SIDECARS_TRACKERS_LABEL = "blob_sidecars_trackers"; static final UInt64 MAX_WAIT_RELATIVE_TO_ATT_DUE_MILLIS = UInt64.valueOf(1500); - static final UInt64 MIN_WAIT_MILLIS = UInt64.valueOf(500); + static final UInt64 MIN_WAIT_MILLIS = UInt64.ZERO; static final UInt64 TARGET_WAIT_MILLIS = UInt64.valueOf(1000); private final SettableLabelledGauge sizeGauge; From 93ab4a8cd8102c724882ef7f2771aad2065ed017 Mon Sep 17 00:00:00 2001 From: Paul Harris Date: Mon, 2 Dec 2024 08:04:07 +1000 Subject: [PATCH 02/20] alter ActiveP2pNetwork concept of close to in sync (#8853) - moved the isCloseToInSync into recentChaindata Signed-off-by: Paul Harris Co-authored-by: Enrico Del Fante --- .../logic/common/util/ForkChoiceUtil.java | 2 +- .../networking/eth2/ActiveEth2P2PNetwork.java | 33 ++++----- .../teku/networking/eth2/Eth2P2PNetwork.java | 2 +- .../eth2/mock/NoOpEth2P2PNetwork.java | 2 +- .../eth2/ActiveEth2P2PNetworkTest.java | 72 ++++++++----------- .../beaconchain/BeaconChainController.java | 3 +- .../services/beaconchain/SlotProcessor.java | 2 +- .../teku/storage/client/RecentChainData.java | 24 +++++++ .../storage/client/RecentChainDataTest.java | 39 ++++++++++ 9 files changed, 112 insertions(+), 67 deletions(-) diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/common/util/ForkChoiceUtil.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/common/util/ForkChoiceUtil.java index 859b5fe6e81..2e6ce3e6d39 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/common/util/ForkChoiceUtil.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/common/util/ForkChoiceUtil.java @@ -68,7 +68,7 @@ public ForkChoiceUtil( } public UInt64 getSlotsSinceGenesis(final ReadOnlyStore store, final boolean useUnixTime) { - UInt64 time = + final UInt64 time = useUnixTime ? UInt64.valueOf(Instant.now().getEpochSecond()) : store.getTimeSeconds(); return getCurrentSlot(time, store.getGenesisTime()); } diff --git a/networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/ActiveEth2P2PNetwork.java b/networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/ActiveEth2P2PNetwork.java index 6dd13b98f27..53fcc248414 100644 --- a/networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/ActiveEth2P2PNetwork.java +++ b/networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/ActiveEth2P2PNetwork.java @@ -130,7 +130,17 @@ private synchronized void startup() { processedAttestationSubscriptionProvider.subscribe(gossipForkManager::publishAttestation); eventChannels.subscribe(BlockGossipChannel.class, gossipForkManager::publishBlock); eventChannels.subscribe(BlobSidecarGossipChannel.class, gossipForkManager::publishBlobSidecar); - if (isCloseToInSync()) { + if (recentChainData.isCloseToInSync()) { + startGossip(); + } + peerManager.subscribeConnect(peer -> onPeerConnected()); + } + + private void onPeerConnected() { + if (gossipStarted.get() || state.get() != State.RUNNING) { + return; + } + if (recentChainData.isCloseToInSync()) { startGossip(); } } @@ -167,36 +177,19 @@ private synchronized void stopGossip() { } @Override - public void onSyncStateChanged(final boolean isInSync, final boolean isOptimistic) { + public void onSyncStateChanged(final boolean isCloseToInSync, final boolean isOptimistic) { gossipForkManager.onOptimisticHeadChanged(isOptimistic); if (state.get() != State.RUNNING) { return; } - if (isInSync || isCloseToInSync()) { + if (isCloseToInSync) { startGossip(); } else { stopGossip(); } } - @VisibleForTesting - boolean isCloseToInSync() { - final Optional currentEpoch = recentChainData.getCurrentEpoch(); - if (currentEpoch.isEmpty()) { - return false; - } - - final int maxLookaheadEpochs = spec.getSpecConfig(currentEpoch.get()).getMaxSeedLookahead(); - final int slotsPerEpoch = spec.slotsPerEpoch(currentEpoch.get()); - final int maxLookaheadSlots = slotsPerEpoch * maxLookaheadEpochs; - - return recentChainData - .getChainHeadSlotsBehind() - .orElse(UInt64.MAX_VALUE) - .isLessThanOrEqualTo(maxLookaheadSlots); - } - private void setTopicScoringParams() { gossipUpdateTask = asyncRunner.runWithFixedDelay( diff --git a/networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/Eth2P2PNetwork.java b/networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/Eth2P2PNetwork.java index 0b7dbc34c49..96dce96b2c5 100644 --- a/networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/Eth2P2PNetwork.java +++ b/networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/Eth2P2PNetwork.java @@ -28,7 +28,7 @@ public interface Eth2P2PNetwork extends P2PNetwork { void onEpoch(UInt64 epoch); - void onSyncStateChanged(final boolean isInSync, final boolean isOptimistic); + void onSyncStateChanged(final boolean isCloseToInSync, final boolean isOptimistic); void subscribeToAttestationSubnetId(int subnetId); diff --git a/networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/mock/NoOpEth2P2PNetwork.java b/networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/mock/NoOpEth2P2PNetwork.java index f80fab25fea..e230b90b5d3 100644 --- a/networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/mock/NoOpEth2P2PNetwork.java +++ b/networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/mock/NoOpEth2P2PNetwork.java @@ -37,7 +37,7 @@ public NoOpEth2P2PNetwork(final Spec spec) { public void onEpoch(final UInt64 epoch) {} @Override - public void onSyncStateChanged(final boolean isInSync, final boolean isOptimistic) {} + public void onSyncStateChanged(final boolean isCloseToInSync, final boolean isOptimistic) {} @Override public void subscribeToAttestationSubnetId(final int subnetId) {} diff --git a/networking/eth2/src/test/java/tech/pegasys/teku/networking/eth2/ActiveEth2P2PNetworkTest.java b/networking/eth2/src/test/java/tech/pegasys/teku/networking/eth2/ActiveEth2P2PNetworkTest.java index 91810bc2cf4..55fdbcd61c3 100644 --- a/networking/eth2/src/test/java/tech/pegasys/teku/networking/eth2/ActiveEth2P2PNetworkTest.java +++ b/networking/eth2/src/test/java/tech/pegasys/teku/networking/eth2/ActiveEth2P2PNetworkTest.java @@ -39,8 +39,10 @@ import tech.pegasys.teku.networking.eth2.gossip.encoding.GossipEncoding; import tech.pegasys.teku.networking.eth2.gossip.forks.GossipForkManager; import tech.pegasys.teku.networking.eth2.gossip.topics.ProcessedAttestationSubscriptionProvider; +import tech.pegasys.teku.networking.eth2.peers.Eth2Peer; import tech.pegasys.teku.networking.eth2.peers.Eth2PeerManager; import tech.pegasys.teku.networking.p2p.discovery.DiscoveryNetwork; +import tech.pegasys.teku.networking.p2p.peer.PeerConnectedSubscriber; import tech.pegasys.teku.spec.Spec; import tech.pegasys.teku.spec.TestSpecFactory; import tech.pegasys.teku.spec.datastructures.attestation.ProcessedAttestationListener; @@ -81,10 +83,6 @@ public class ActiveEth2P2PNetworkTest { private Fork altairFork; private Bytes32 genesisValidatorsRoot; - private final int maxFollowDistanceSlots = - spec.getGenesisSpecConfig().getMaxSeedLookahead() - * spec.slotsPerEpoch(storageSystem.combinedChainDataClient().getCurrentEpoch()); - @BeforeEach public void setup() { when(discoveryNetwork.start()).thenReturn(SafeFuture.completedFuture(null)); @@ -190,14 +188,39 @@ public void unsubscribeFromSyncCommitteeSubnetId_shouldUpdateDiscoveryENR() { } @Test - void onSyncStateChanged_shouldEnableGossipWhenInSync() { + void shouldStartGossipOnPeerConnect() { + @SuppressWarnings("unchecked") + final ArgumentCaptor> peerManagerCaptor = + ArgumentCaptor.forClass(PeerConnectedSubscriber.class); // Current slot is a long way beyond the chain head - storageSystem.chainUpdater().setCurrentSlot(UInt64.valueOf(1000)); + storageSystem.chainUpdater().setCurrentSlot(UInt64.valueOf(64)); assertThat(network.start()).isCompleted(); - // Won't start gossip as chain head is too old + verify(peerManager).subscribeConnect(peerManagerCaptor.capture()); + + network.onSyncStateChanged(false, false); + // based on network time we know we're too far behind, so we don't start gossip + verify(gossipForkManager, never()).configureGossipForEpoch(any()); + + // we are still too far behind, so on peer connect gossip is not started + peerManagerCaptor.getValue().onConnected(mock(Eth2Peer.class)); verify(gossipForkManager, never()).configureGossipForEpoch(any()); + // Advance the chain + storageSystem.chainUpdater().updateBestBlock(storageSystem.chainUpdater().advanceChain(64)); + + // on peer connect gossip is started + peerManagerCaptor.getValue().onConnected(mock(Eth2Peer.class)); + verify(gossipForkManager).configureGossipForEpoch(any()); + } + + @Test + void onSyncStateChanged_shouldEnableGossipWhenInSync() { + // Current slot is a long way beyond the chain head + storageSystem.chainUpdater().setCurrentSlot(UInt64.valueOf(32)); + + assertThat(network.start()).isCompleted(); + network.onSyncStateChanged(true, false); // Even though we're a long way behind, start gossip because we believe we're in sync @@ -259,41 +282,6 @@ void onSyncStateChanged_shouldNotResultInMultipleSubscriptions() { verify(eventChannels, times(1)).subscribe(eq(BlockGossipChannel.class), any()); } - @Test - void isCloseToInSync_shouldCalculateWhenDistanceOutOfRange() { - storageSystem.chainUpdater().setCurrentSlot(UInt64.valueOf(maxFollowDistanceSlots + 1)); - assertThat(network.isCloseToInSync()).isFalse(); - } - - @Test - void isCloseToInSync_shouldCalculateWhenDistanceInRange() { - storageSystem.chainUpdater().setCurrentSlot(UInt64.valueOf(maxFollowDistanceSlots)); - assertThat(network.isCloseToInSync()).isTrue(); - } - - @Test - void isCloseToInSync_shouldReturnFalseWhenEmptyCurrentEpoch() { - final StorageSystem storageSystem = InMemoryStorageSystemBuilder.buildDefault(); - final RecentChainData recentChainData = storageSystem.recentChainData(); - final ActiveEth2P2PNetwork network = - new ActiveEth2P2PNetwork( - spec, - asyncRunner, - discoveryNetwork, - peerManager, - gossipForkManager, - eventChannels, - recentChainData, - attestationSubnetService, - syncCommitteeSubnetService, - gossipEncoding, - gossipConfigurator, - processedAttestationSubscriptionProvider, - true); - - assertThat(network.isCloseToInSync()).isFalse(); - } - @SuppressWarnings("unchecked") private ArgumentCaptor> subnetIdCaptor() { return ArgumentCaptor.forClass(Iterable.class); diff --git a/services/beaconchain/src/main/java/tech/pegasys/teku/services/beaconchain/BeaconChainController.java b/services/beaconchain/src/main/java/tech/pegasys/teku/services/beaconchain/BeaconChainController.java index 5ceac99ab19..47e0d1540e4 100644 --- a/services/beaconchain/src/main/java/tech/pegasys/teku/services/beaconchain/BeaconChainController.java +++ b/services/beaconchain/src/main/java/tech/pegasys/teku/services/beaconchain/BeaconChainController.java @@ -1326,7 +1326,8 @@ public void initSyncService() { // p2pNetwork subscription so gossip can be enabled and disabled appropriately syncService.subscribeToSyncStateChangesAndUpdate( - state -> p2pNetwork.onSyncStateChanged(state.isInSync(), state.isOptimistic())); + state -> + p2pNetwork.onSyncStateChanged(recentChainData.isCloseToInSync(), state.isOptimistic())); } protected void initOperationsReOrgManager() { diff --git a/services/beaconchain/src/main/java/tech/pegasys/teku/services/beaconchain/SlotProcessor.java b/services/beaconchain/src/main/java/tech/pegasys/teku/services/beaconchain/SlotProcessor.java index 752ddba91fb..75521aece80 100644 --- a/services/beaconchain/src/main/java/tech/pegasys/teku/services/beaconchain/SlotProcessor.java +++ b/services/beaconchain/src/main/java/tech/pegasys/teku/services/beaconchain/SlotProcessor.java @@ -151,7 +151,7 @@ private void processEpochPrecompute(final UInt64 epoch) { } private void processSlotWhileSyncing(final SyncState currentSyncState) { - UInt64 slot = nodeSlot.getValue(); + final UInt64 slot = nodeSlot.getValue(); this.forkChoiceTrigger.onSlotStartedWhileSyncing(slot); if (currentSyncState == SyncState.AWAITING_EL) { eventLog.syncEventAwaitingEL(slot, recentChainData.getHeadSlot(), p2pNetwork.getPeerCount()); diff --git a/storage/src/main/java/tech/pegasys/teku/storage/client/RecentChainData.java b/storage/src/main/java/tech/pegasys/teku/storage/client/RecentChainData.java index 4b274001d9a..9c18c758156 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/client/RecentChainData.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/client/RecentChainData.java @@ -15,6 +15,7 @@ import static tech.pegasys.teku.infrastructure.time.TimeUtilities.secondsToMillis; +import com.google.common.annotations.VisibleForTesting; import java.util.Collections; import java.util.List; import java.util.Map; @@ -41,6 +42,7 @@ import tech.pegasys.teku.spec.Spec; import tech.pegasys.teku.spec.SpecMilestone; import tech.pegasys.teku.spec.SpecVersion; +import tech.pegasys.teku.spec.config.SpecConfig; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; import tech.pegasys.teku.spec.datastructures.blocks.BeaconBlock; import tech.pegasys.teku.spec.datastructures.blocks.MinimalBeaconBlockSummary; @@ -58,6 +60,7 @@ import tech.pegasys.teku.spec.datastructures.state.ForkInfo; import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconState; import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconStateCache; +import tech.pegasys.teku.spec.logic.common.helpers.MiscHelpers; import tech.pegasys.teku.spec.logic.common.util.BeaconStateUtil; import tech.pegasys.teku.storage.api.ChainHeadChannel; import tech.pegasys.teku.storage.api.FinalizedCheckpointChannel; @@ -194,6 +197,27 @@ public UInt64 computeTimeAtSlot(final UInt64 slot) { return genesisTime.plus(slot.times(spec.getSecondsPerSlot(slot))); } + @VisibleForTesting + boolean isCloseToInSync(final UInt64 currentTimeSeconds) { + final SpecVersion specVersion = spec.getGenesisSpec(); + final MiscHelpers miscHelpers = specVersion.miscHelpers(); + final SpecConfig specConfig = specVersion.getConfig(); + final UInt64 networkSlot = miscHelpers.computeSlotAtTime(getGenesisTime(), currentTimeSeconds); + + final int maxLookaheadEpochs = specConfig.getMaxSeedLookahead(); + final int slotsPerEpoch = specVersion.getSlotsPerEpoch(); + final int maxLookaheadSlots = slotsPerEpoch * maxLookaheadEpochs; + + return networkSlot.minusMinZero(getHeadSlot()).isLessThanOrEqualTo(maxLookaheadSlots); + } + + public boolean isCloseToInSync() { + if (store == null) { + return false; + } + return isCloseToInSync(store.getTimeInSeconds()); + } + public Optional getGenesisData() { return genesisData; } diff --git a/storage/src/test/java/tech/pegasys/teku/storage/client/RecentChainDataTest.java b/storage/src/test/java/tech/pegasys/teku/storage/client/RecentChainDataTest.java index c040f5e291b..ae88e4465d1 100644 --- a/storage/src/test/java/tech/pegasys/teku/storage/client/RecentChainDataTest.java +++ b/storage/src/test/java/tech/pegasys/teku/storage/client/RecentChainDataTest.java @@ -752,6 +752,45 @@ public void getBlockRootBySlotWithHeadRoot_forUnknownHeadRoot() { assertThat(recentChainData.getBlockRootInEffectBySlot(bestBlock.getSlot(), headRoot)).isEmpty(); } + @Test + public void isCloseToInSync_preGenesis() { + initPreGenesis(); + assertThat(recentChainData.isCloseToInSync()).isFalse(); + } + + @Test + public void isCloseToSync_belowBoundary() { + initPostGenesis(); + final SpecConfig specConfig = spec.getGenesisSpecConfig(); + final int seconds = + specConfig.getMaxSeedLookahead() + * specConfig.getSlotsPerEpoch() + * specConfig.getSecondsPerSlot(); + assertThat(recentChainData.isCloseToInSync(UInt64.valueOf(seconds - 1))).isTrue(); + } + + @Test + public void isCloseToSync_atBoundary() { + initPostGenesis(); + final SpecConfig specConfig = spec.getGenesisSpecConfig(); + final int seconds = + specConfig.getMaxSeedLookahead() + * specConfig.getSlotsPerEpoch() + * specConfig.getSecondsPerSlot(); + assertThat(recentChainData.isCloseToInSync(UInt64.valueOf(seconds))).isTrue(); + } + + @Test + public void isCloseToSync_aboveBoundary() { + initPostGenesis(); + final SpecConfig specConfig = spec.getGenesisSpecConfig(); + final int seconds = + specConfig.getMaxSeedLookahead() + * specConfig.getSlotsPerEpoch() + * specConfig.getSecondsPerSlot(); + assertThat(recentChainData.isCloseToInSync(UInt64.valueOf(seconds + 8))).isFalse(); + } + @Test public void getBlockRootBySlotWithHeadRoot_withForkRoot() { initPostGenesis(); From 6f78534a72b7d2a339f8fc43cac856095e943698 Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Mon, 2 Dec 2024 12:55:42 +0100 Subject: [PATCH 03/20] Provide `SingleAttestation` datastructure (#8867) --- .../operations/Attestation.java | 21 ++++- .../operations/AttestationSchema.java | 9 ++ .../operations/SingleAttestation.java | 93 +++++++++++++++++++ .../operations/SingleAttestationSchema.java | 84 +++++++++++++++++ .../versions/electra/AttestationElectra.java | 14 --- .../versions/phase0/AttestationPhase0.java | 14 --- .../schemas/SchemaDefinitionsElectra.java | 10 ++ .../registry/SchemaRegistryBuilder.java | 11 ++- .../spec/schemas/registry/SchemaTypes.java | 3 + .../SingleAttestationPropertyTest.java | 37 ++++++++ .../operations/SingleAttestationSupplier.java | 25 +++++ .../teku/spec/util/DataStructureUtil.java | 9 ++ 12 files changed, 299 insertions(+), 31 deletions(-) create mode 100644 ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/operations/SingleAttestation.java create mode 100644 ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/operations/SingleAttestationSchema.java create mode 100644 ethereum/spec/src/property-test/java/tech/pegasys/teku/spec/datastructures/operations/SingleAttestationPropertyTest.java create mode 100644 ethereum/spec/src/testFixtures/java/tech/pegasys/teku/spec/propertytest/suppliers/operations/SingleAttestationSupplier.java diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/operations/Attestation.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/operations/Attestation.java index 57f0ff3f234..06cef12df1d 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/operations/Attestation.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/operations/Attestation.java @@ -13,6 +13,7 @@ package tech.pegasys.teku.spec.datastructures.operations; +import com.google.common.collect.Sets; import java.util.Collection; import java.util.List; import java.util.Optional; @@ -34,9 +35,13 @@ public interface Attestation extends SszContainer { @Override AttestationSchema getSchema(); - UInt64 getEarliestSlotForForkChoiceProcessing(final Spec spec); + default UInt64 getEarliestSlotForForkChoiceProcessing(final Spec spec) { + return getData().getEarliestSlotForForkChoice(spec); + } - Collection getDependentBlockRoots(); + default Collection getDependentBlockRoots() { + return Sets.newHashSet(getData().getTarget().getRoot(), getData().getBeaconBlockRoot()); + } AttestationData getData(); @@ -65,4 +70,16 @@ default List getCommitteeIndicesRequired() { } boolean requiresCommitteeBits(); + + default boolean isSingleAttestation() { + return false; + } + + default SingleAttestation toSingleAttestationRequired() { + throw new UnsupportedOperationException("Not a SingleAttestation"); + } + + default UInt64 getValidatorIndexRequired() { + throw new UnsupportedOperationException("Not a SingleAttestation"); + } } diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/operations/AttestationSchema.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/operations/AttestationSchema.java index 3b0b0f880ec..f3b45e20c0f 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/operations/AttestationSchema.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/operations/AttestationSchema.java @@ -41,6 +41,11 @@ default SszBitlist createEmptyAggregationBits() { return bitsSchema.ofBits(Math.toIntExact(bitsSchema.getMaxLength())); } + default SszBitlist createAggregationBitsOf(final int size, final int... indices) { + final SszBitlistSchema bitsSchema = getAggregationBitsSchema(); + return bitsSchema.ofBits(size, indices); + } + default Optional createEmptyCommitteeBits() { return getCommitteeBitsSchema().map(SszBitvectorSchema::ofBits); } @@ -54,6 +59,10 @@ default Optional toVersionElectra() { return Optional.empty(); } + default SingleAttestationSchema toSingleAttestationSchemaRequired() { + throw new UnsupportedOperationException("Not a SingleAttestationSchema"); + } + SszBitlistSchema getAggregationBitsSchema(); Optional> getCommitteeBitsSchema(); diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/operations/SingleAttestation.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/operations/SingleAttestation.java new file mode 100644 index 00000000000..5aaaa1c73ac --- /dev/null +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/operations/SingleAttestation.java @@ -0,0 +1,93 @@ +/* + * Copyright Consensys Software Inc., 2024 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.spec.datastructures.operations; + +import tech.pegasys.teku.bls.BLSSignature; +import tech.pegasys.teku.infrastructure.ssz.collections.SszBitlist; +import tech.pegasys.teku.infrastructure.ssz.containers.Container4; +import tech.pegasys.teku.infrastructure.ssz.primitive.SszUInt64; +import tech.pegasys.teku.infrastructure.ssz.tree.TreeNode; +import tech.pegasys.teku.infrastructure.unsigned.UInt64; +import tech.pegasys.teku.spec.datastructures.type.SszSignature; + +public class SingleAttestation + extends Container4 + implements Attestation { + public SingleAttestation(final SingleAttestationSchema type, final TreeNode backingNode) { + super(type, backingNode); + } + + public SingleAttestation( + final SingleAttestationSchema schema, + final UInt64 committeeIndex, + final UInt64 validatorIndex, + final AttestationData data, + final BLSSignature signature) { + super( + schema, + SszUInt64.of(committeeIndex), + SszUInt64.of(validatorIndex), + data, + new SszSignature(signature)); + } + + @Override + public SingleAttestationSchema getSchema() { + return (SingleAttestationSchema) super.getSchema(); + } + + @Override + public AttestationData getData() { + return getField2(); + } + + @Override + public SszBitlist getAggregationBits() { + throw new UnsupportedOperationException("Not supported in SingleAttestation"); + } + + @Override + public UInt64 getFirstCommitteeIndex() { + return getField0().get(); + } + + @Override + public BLSSignature getAggregateSignature() { + return getField3().getSignature(); + } + + public BLSSignature getSignature() { + return getField3().getSignature(); + } + + @Override + public boolean requiresCommitteeBits() { + return false; + } + + @Override + public boolean isSingleAttestation() { + return true; + } + + @Override + public SingleAttestation toSingleAttestationRequired() { + return this; + } + + @Override + public UInt64 getValidatorIndexRequired() { + return getField1().get(); + } +} diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/operations/SingleAttestationSchema.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/operations/SingleAttestationSchema.java new file mode 100644 index 00000000000..0993ec88e7f --- /dev/null +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/operations/SingleAttestationSchema.java @@ -0,0 +1,84 @@ +/* + * Copyright Consensys Software Inc., 2024 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.spec.datastructures.operations; + +import java.util.Optional; +import java.util.function.Supplier; +import tech.pegasys.teku.bls.BLSSignature; +import tech.pegasys.teku.infrastructure.ssz.collections.SszBitlist; +import tech.pegasys.teku.infrastructure.ssz.collections.SszBitvector; +import tech.pegasys.teku.infrastructure.ssz.containers.ContainerSchema4; +import tech.pegasys.teku.infrastructure.ssz.primitive.SszUInt64; +import tech.pegasys.teku.infrastructure.ssz.schema.SszPrimitiveSchemas; +import tech.pegasys.teku.infrastructure.ssz.schema.collections.SszBitlistSchema; +import tech.pegasys.teku.infrastructure.ssz.schema.collections.SszBitvectorSchema; +import tech.pegasys.teku.infrastructure.ssz.tree.TreeNode; +import tech.pegasys.teku.infrastructure.unsigned.UInt64; +import tech.pegasys.teku.spec.datastructures.type.SszSignature; +import tech.pegasys.teku.spec.datastructures.type.SszSignatureSchema; + +public class SingleAttestationSchema + extends ContainerSchema4 + implements AttestationSchema { + public SingleAttestationSchema() { + super( + "SingleAttestation", + namedSchema("committee_index", SszPrimitiveSchemas.UINT64_SCHEMA), + namedSchema("attester_index", SszPrimitiveSchemas.UINT64_SCHEMA), + namedSchema("data", AttestationData.SSZ_SCHEMA), + namedSchema("signature", SszSignatureSchema.INSTANCE)); + } + + @Override + public SingleAttestation createFromBackingNode(final TreeNode node) { + return new SingleAttestation(this, node); + } + + public SingleAttestation create( + final UInt64 committeeIndex, + final UInt64 attesterIndex, + final AttestationData data, + final BLSSignature signature) { + return new SingleAttestation(this, committeeIndex, attesterIndex, data, signature); + } + + @Override + public Attestation create( + final SszBitlist aggregationBits, + final AttestationData data, + final BLSSignature signature, + final Supplier committeeBits) { + throw new UnsupportedOperationException("Not supported in SingleAttestation"); + } + + @Override + public SszBitlistSchema getAggregationBitsSchema() { + throw new UnsupportedOperationException("Not supported in SingleAttestation"); + } + + @Override + public Optional> getCommitteeBitsSchema() { + return Optional.empty(); + } + + @Override + public SingleAttestationSchema toSingleAttestationSchemaRequired() { + return this; + } + + @Override + public boolean requiresCommitteeBits() { + return false; + } +} diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/operations/versions/electra/AttestationElectra.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/operations/versions/electra/AttestationElectra.java index 1842c46e897..cebb76f5cb3 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/operations/versions/electra/AttestationElectra.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/operations/versions/electra/AttestationElectra.java @@ -13,18 +13,14 @@ package tech.pegasys.teku.spec.datastructures.operations.versions.electra; -import com.google.common.collect.Sets; -import java.util.Collection; import java.util.List; import java.util.Optional; -import org.apache.tuweni.bytes.Bytes32; import tech.pegasys.teku.bls.BLSSignature; import tech.pegasys.teku.infrastructure.ssz.collections.SszBitlist; import tech.pegasys.teku.infrastructure.ssz.collections.SszBitvector; import tech.pegasys.teku.infrastructure.ssz.containers.Container4; import tech.pegasys.teku.infrastructure.ssz.tree.TreeNode; import tech.pegasys.teku.infrastructure.unsigned.UInt64; -import tech.pegasys.teku.spec.Spec; import tech.pegasys.teku.spec.datastructures.operations.Attestation; import tech.pegasys.teku.spec.datastructures.operations.AttestationData; import tech.pegasys.teku.spec.datastructures.type.SszSignature; @@ -51,16 +47,6 @@ public AttestationElectraSchema getSchema() { return (AttestationElectraSchema) super.getSchema(); } - @Override - public UInt64 getEarliestSlotForForkChoiceProcessing(final Spec spec) { - return getData().getEarliestSlotForForkChoice(spec); - } - - @Override - public Collection getDependentBlockRoots() { - return Sets.newHashSet(getData().getTarget().getRoot(), getData().getBeaconBlockRoot()); - } - @Override public SszBitlist getAggregationBits() { return getField0(); diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/operations/versions/phase0/AttestationPhase0.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/operations/versions/phase0/AttestationPhase0.java index dd343f37361..e7d464a43aa 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/operations/versions/phase0/AttestationPhase0.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/operations/versions/phase0/AttestationPhase0.java @@ -13,15 +13,11 @@ package tech.pegasys.teku.spec.datastructures.operations.versions.phase0; -import com.google.common.collect.Sets; -import java.util.Collection; -import org.apache.tuweni.bytes.Bytes32; import tech.pegasys.teku.bls.BLSSignature; import tech.pegasys.teku.infrastructure.ssz.collections.SszBitlist; import tech.pegasys.teku.infrastructure.ssz.containers.Container3; import tech.pegasys.teku.infrastructure.ssz.tree.TreeNode; import tech.pegasys.teku.infrastructure.unsigned.UInt64; -import tech.pegasys.teku.spec.Spec; import tech.pegasys.teku.spec.datastructures.operations.Attestation; import tech.pegasys.teku.spec.datastructures.operations.AttestationData; import tech.pegasys.teku.spec.datastructures.type.SszSignature; @@ -47,16 +43,6 @@ public AttestationPhase0Schema getSchema() { return (AttestationPhase0Schema) super.getSchema(); } - @Override - public UInt64 getEarliestSlotForForkChoiceProcessing(final Spec spec) { - return getData().getEarliestSlotForForkChoice(spec); - } - - @Override - public Collection getDependentBlockRoots() { - return Sets.newHashSet(getData().getTarget().getRoot(), getData().getBeaconBlockRoot()); - } - @Override public SszBitlist getAggregationBits() { return getField0(); diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/schemas/SchemaDefinitionsElectra.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/schemas/SchemaDefinitionsElectra.java index 46ef4ecd760..2888b00b1ff 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/schemas/SchemaDefinitionsElectra.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/schemas/SchemaDefinitionsElectra.java @@ -20,6 +20,7 @@ import static tech.pegasys.teku.spec.schemas.registry.SchemaTypes.PENDING_CONSOLIDATIONS_SCHEMA; import static tech.pegasys.teku.spec.schemas.registry.SchemaTypes.PENDING_DEPOSITS_SCHEMA; import static tech.pegasys.teku.spec.schemas.registry.SchemaTypes.PENDING_PARTIAL_WITHDRAWALS_SCHEMA; +import static tech.pegasys.teku.spec.schemas.registry.SchemaTypes.SINGLE_ATTESTATION_SCHEMA; import static tech.pegasys.teku.spec.schemas.registry.SchemaTypes.WITHDRAWAL_REQUEST_SCHEMA; import java.util.Optional; @@ -31,6 +32,7 @@ import tech.pegasys.teku.spec.datastructures.execution.versions.electra.DepositRequestSchema; import tech.pegasys.teku.spec.datastructures.execution.versions.electra.ExecutionRequestsSchema; import tech.pegasys.teku.spec.datastructures.execution.versions.electra.WithdrawalRequestSchema; +import tech.pegasys.teku.spec.datastructures.operations.SingleAttestationSchema; import tech.pegasys.teku.spec.datastructures.state.versions.electra.PendingConsolidation; import tech.pegasys.teku.spec.datastructures.state.versions.electra.PendingConsolidation.PendingConsolidationSchema; import tech.pegasys.teku.spec.datastructures.state.versions.electra.PendingDeposit; @@ -53,6 +55,8 @@ public class SchemaDefinitionsElectra extends SchemaDefinitionsDeneb { private final PendingPartialWithdrawalSchema pendingPartialWithdrawalSchema; private final PendingConsolidationSchema pendingConsolidationSchema; + private final SingleAttestationSchema singleAttestationSchema; + public SchemaDefinitionsElectra(final SchemaRegistry schemaRegistry) { super(schemaRegistry); this.executionRequestsSchema = schemaRegistry.get(EXECUTION_REQUESTS_SCHEMA); @@ -60,6 +64,8 @@ public SchemaDefinitionsElectra(final SchemaRegistry schemaRegistry) { this.pendingPartialWithdrawalsSchema = schemaRegistry.get(PENDING_PARTIAL_WITHDRAWALS_SCHEMA); this.pendingConsolidationsSchema = schemaRegistry.get(PENDING_CONSOLIDATIONS_SCHEMA); + this.singleAttestationSchema = schemaRegistry.get(SINGLE_ATTESTATION_SCHEMA); + this.depositRequestSchema = schemaRegistry.get(DEPOSIT_REQUEST_SCHEMA); this.withdrawalRequestSchema = schemaRegistry.get(WITHDRAWAL_REQUEST_SCHEMA); this.consolidationRequestSchema = schemaRegistry.get(CONSOLIDATION_REQUEST_SCHEMA); @@ -126,6 +132,10 @@ public PendingConsolidation.PendingConsolidationSchema getPendingConsolidationSc return pendingPartialWithdrawalsSchema; } + public SingleAttestationSchema getSingleAttestationSchema() { + return singleAttestationSchema; + } + public SszListSchema getPendingConsolidationsSchema() { return pendingConsolidationsSchema; } diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/schemas/registry/SchemaRegistryBuilder.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/schemas/registry/SchemaRegistryBuilder.java index 0f459fb0d4b..16ac090a73a 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/schemas/registry/SchemaRegistryBuilder.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/schemas/registry/SchemaRegistryBuilder.java @@ -58,6 +58,7 @@ import static tech.pegasys.teku.spec.schemas.registry.SchemaTypes.SIGNED_BLOCK_CONTENTS_SCHEMA; import static tech.pegasys.teku.spec.schemas.registry.SchemaTypes.SIGNED_BLS_TO_EXECUTION_CHANGE_SCHEMA; import static tech.pegasys.teku.spec.schemas.registry.SchemaTypes.SIGNED_BUILDER_BID_SCHEMA; +import static tech.pegasys.teku.spec.schemas.registry.SchemaTypes.SINGLE_ATTESTATION_SCHEMA; import static tech.pegasys.teku.spec.schemas.registry.SchemaTypes.SYNCNETS_ENR_FIELD_SCHEMA; import static tech.pegasys.teku.spec.schemas.registry.SchemaTypes.WITHDRAWAL_REQUEST_SCHEMA; import static tech.pegasys.teku.spec.schemas.registry.SchemaTypes.WITHDRAWAL_SCHEMA; @@ -117,6 +118,7 @@ import tech.pegasys.teku.spec.datastructures.operations.IndexedAttestationSchema; import tech.pegasys.teku.spec.datastructures.operations.SignedAggregateAndProof.SignedAggregateAndProofSchema; import tech.pegasys.teku.spec.datastructures.operations.SignedBlsToExecutionChangeSchema; +import tech.pegasys.teku.spec.datastructures.operations.SingleAttestationSchema; import tech.pegasys.teku.spec.datastructures.operations.versions.electra.AttestationElectraSchema; import tech.pegasys.teku.spec.datastructures.operations.versions.phase0.AttestationPhase0Schema; import tech.pegasys.teku.spec.datastructures.state.HistoricalBatch.HistoricalBatchSchema; @@ -188,7 +190,14 @@ public static SchemaRegistryBuilder create() { .addProvider(createDepositRequestSchemaProvider()) .addProvider(createWithdrawalRequestSchemaProvider()) .addProvider(createConsolidationRequestSchemaProvider()) - .addProvider(createExecutionRequestsSchemaProvider()); + .addProvider(createExecutionRequestsSchemaProvider()) + .addProvider(createSingleAttestationSchemaProvider()); + } + + private static SchemaProvider createSingleAttestationSchemaProvider() { + return providerBuilder(SINGLE_ATTESTATION_SCHEMA) + .withCreator(ELECTRA, (registry, specConfig, schemaName) -> new SingleAttestationSchema()) + .build(); } private static SchemaProvider createDepositRequestSchemaProvider() { diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/schemas/registry/SchemaTypes.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/schemas/registry/SchemaTypes.java index 4a8356d373a..1c6b3756672 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/schemas/registry/SchemaTypes.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/schemas/registry/SchemaTypes.java @@ -59,6 +59,7 @@ import tech.pegasys.teku.spec.datastructures.operations.IndexedAttestationSchema; import tech.pegasys.teku.spec.datastructures.operations.SignedAggregateAndProof.SignedAggregateAndProofSchema; import tech.pegasys.teku.spec.datastructures.operations.SignedBlsToExecutionChangeSchema; +import tech.pegasys.teku.spec.datastructures.operations.SingleAttestationSchema; import tech.pegasys.teku.spec.datastructures.state.HistoricalBatch.HistoricalBatchSchema; import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconState; import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconStateSchema; @@ -169,6 +170,8 @@ public class SchemaTypes { create("WITHDRAWAL_REQUEST_SCHEMA"); public static final SchemaId CONSOLIDATION_REQUEST_SCHEMA = create("CONSOLIDATION_REQUEST_SCHEMA"); + public static final SchemaId SINGLE_ATTESTATION_SCHEMA = + create("SINGLE_ATTESTATION_SCHEMA"); private SchemaTypes() { // Prevent instantiation diff --git a/ethereum/spec/src/property-test/java/tech/pegasys/teku/spec/datastructures/operations/SingleAttestationPropertyTest.java b/ethereum/spec/src/property-test/java/tech/pegasys/teku/spec/datastructures/operations/SingleAttestationPropertyTest.java new file mode 100644 index 00000000000..ea0c7f944ce --- /dev/null +++ b/ethereum/spec/src/property-test/java/tech/pegasys/teku/spec/datastructures/operations/SingleAttestationPropertyTest.java @@ -0,0 +1,37 @@ +/* + * Copyright Consensys Software Inc., 2022 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.spec.datastructures.operations; + +import static tech.pegasys.teku.spec.propertytest.util.PropertyTestHelper.assertDeserializeMutatedThrowsExpected; +import static tech.pegasys.teku.spec.propertytest.util.PropertyTestHelper.assertRoundTrip; + +import com.fasterxml.jackson.core.JsonProcessingException; +import net.jqwik.api.ForAll; +import net.jqwik.api.Property; +import tech.pegasys.teku.spec.propertytest.suppliers.operations.SingleAttestationSupplier; + +public class SingleAttestationPropertyTest { + @Property + void roundTrip(@ForAll(supplier = SingleAttestationSupplier.class) final Attestation attestation) + throws JsonProcessingException { + assertRoundTrip(attestation); + } + + @Property + void deserializeMutated( + @ForAll(supplier = SingleAttestationSupplier.class) final Attestation attestation, + @ForAll final int seed) { + assertDeserializeMutatedThrowsExpected(attestation, seed); + } +} diff --git a/ethereum/spec/src/testFixtures/java/tech/pegasys/teku/spec/propertytest/suppliers/operations/SingleAttestationSupplier.java b/ethereum/spec/src/testFixtures/java/tech/pegasys/teku/spec/propertytest/suppliers/operations/SingleAttestationSupplier.java new file mode 100644 index 00000000000..de6215d748e --- /dev/null +++ b/ethereum/spec/src/testFixtures/java/tech/pegasys/teku/spec/propertytest/suppliers/operations/SingleAttestationSupplier.java @@ -0,0 +1,25 @@ +/* + * Copyright Consensys Software Inc., 2022 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.spec.propertytest.suppliers.operations; + +import tech.pegasys.teku.spec.SpecMilestone; +import tech.pegasys.teku.spec.datastructures.operations.SingleAttestation; +import tech.pegasys.teku.spec.propertytest.suppliers.DataStructureUtilSupplier; +import tech.pegasys.teku.spec.util.DataStructureUtil; + +public class SingleAttestationSupplier extends DataStructureUtilSupplier { + public SingleAttestationSupplier() { + super(DataStructureUtil::randomSingleAttestation, SpecMilestone.ELECTRA); + } +} diff --git a/ethereum/spec/src/testFixtures/java/tech/pegasys/teku/spec/util/DataStructureUtil.java b/ethereum/spec/src/testFixtures/java/tech/pegasys/teku/spec/util/DataStructureUtil.java index c89c94d15aa..b1f39032a9a 100644 --- a/ethereum/spec/src/testFixtures/java/tech/pegasys/teku/spec/util/DataStructureUtil.java +++ b/ethereum/spec/src/testFixtures/java/tech/pegasys/teku/spec/util/DataStructureUtil.java @@ -160,6 +160,7 @@ import tech.pegasys.teku.spec.datastructures.operations.SignedAggregateAndProof; import tech.pegasys.teku.spec.datastructures.operations.SignedBlsToExecutionChange; import tech.pegasys.teku.spec.datastructures.operations.SignedVoluntaryExit; +import tech.pegasys.teku.spec.datastructures.operations.SingleAttestation; import tech.pegasys.teku.spec.datastructures.operations.VoluntaryExit; import tech.pegasys.teku.spec.datastructures.operations.versions.altair.ContributionAndProof; import tech.pegasys.teku.spec.datastructures.operations.versions.altair.SignedContributionAndProof; @@ -819,6 +820,14 @@ public Attestation randomAttestation() { this::randomCommitteeBitvector); } + public SingleAttestation randomSingleAttestation() { + return spec.getGenesisSchemaDefinitions() + .toVersionElectra() + .orElseThrow() + .getSingleAttestationSchema() + .create(randomUInt64(), randomUInt64(), randomAttestationData(), randomSignature()); + } + public Attestation randomAttestation(final long slot) { return randomAttestation(UInt64.valueOf(slot)); } From 5aa4bd83d6c96eb8a980b046480ade30bd0fffcc Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Mon, 2 Dec 2024 16:54:15 +0100 Subject: [PATCH 04/20] Enable publish blobs after block by default (#8868) --- CHANGELOG.md | 1 + .../main/java/tech/pegasys/teku/networking/eth2/P2PConfig.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e0c6f219ec..226335a30bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ ### Additions and Improvements - improve block publishing performance, especially relevant with locally produced blocks +- delay blobs publishing until the block is published to at least 1 peer, especially relevant with locally produced blocks with low upload bandwidth connections. Can be disabled via `--Xp2p-gossip-blobs-after-block-enabled=false` ### Bug Fixes - Added a startup script for unix systems to ensure that when jemalloc is installed the script sets the LD_PRELOAD environment variable to the use the jemalloc library diff --git a/networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/P2PConfig.java b/networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/P2PConfig.java index 8a422fa767c..74d43c7c50a 100644 --- a/networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/P2PConfig.java +++ b/networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/P2PConfig.java @@ -38,7 +38,7 @@ public class P2PConfig { public static final int DEFAULT_P2P_TARGET_SUBNET_SUBSCRIBER_COUNT = 2; public static final boolean DEFAULT_SUBSCRIBE_ALL_SUBNETS_ENABLED = false; public static final boolean DEFAULT_GOSSIP_SCORING_ENABLED = true; - public static final boolean DEFAULT_GOSSIP_BLOBS_AFTER_BLOCK_ENABLED = false; + public static final boolean DEFAULT_GOSSIP_BLOBS_AFTER_BLOCK_ENABLED = true; public static final int DEFAULT_BATCH_VERIFY_MAX_THREADS = Math.max(2, Runtime.getRuntime().availableProcessors() / 2); public static final int DEFAULT_BATCH_VERIFY_QUEUE_CAPACITY = 15_000; From a4d9efe9c5f6d963956b83dae8554fcccfe244da Mon Sep 17 00:00:00 2001 From: Lucas Saldanha Date: Tue, 3 Dec 2024 18:55:10 +1300 Subject: [PATCH 05/20] Logic to handle new validators during epoch processing (#8874) --- .../epoch/AbstractEpochProcessor.java | 60 +++++++++++---- .../AbstractValidatorStatusFactory.java | 11 +++ .../epoch/status/ValidatorStatusFactory.java | 15 ++++ .../epoch/EpochProcessorElectra.java | 5 ++ .../epoch/AbstractEpochProcessorTest.java | 73 +++++++++++++++++-- .../AbstractValidatorStatusFactoryTest.java | 69 ++++++++++++++++++ .../epoch/EpochProcessorElectraTest.java | 32 ++++++++ 7 files changed, 247 insertions(+), 18 deletions(-) create mode 100644 ethereum/spec/src/test/java/tech/pegasys/teku/spec/logic/versions/electra/statetransition/epoch/EpochProcessorElectraTest.java diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/common/statetransition/epoch/AbstractEpochProcessor.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/common/statetransition/epoch/AbstractEpochProcessor.java index 6ca58ced844..0a79bbd35f5 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/common/statetransition/epoch/AbstractEpochProcessor.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/common/statetransition/epoch/AbstractEpochProcessor.java @@ -15,6 +15,8 @@ import static tech.pegasys.teku.infrastructure.unsigned.UInt64.ZERO; +import com.google.common.annotations.VisibleForTesting; +import java.util.ArrayList; import java.util.List; import java.util.function.Consumer; import java.util.function.Supplier; @@ -104,18 +106,10 @@ public BeaconState processEpoch(final BeaconState preState) throws EpochProcessi protected void processEpoch(final BeaconState preState, final MutableBeaconState state) throws EpochProcessingException { - /* - WARNING: After Electra, it is possible that the validator set is updated within epoch processing - (process_pending_deposits). This means that the validator set in the state can get out of sync with - our validatorStatuses cache. This is not a problem for the current epoch processing, but it can cause - undesired side effects in the future. - - Up until Electra, the only function that uses validatorStatuses after process_pending_deposits is - process_effective_balance_updates, and in this particular case it is ok that we don't have the new validators - in validatorStatuses. - */ - final ValidatorStatuses validatorStatuses = - validatorStatusFactory.createValidatorStatuses(preState); + // After Electra, it is possible that the validator set is updated within epoch processing + // (process_pending_deposits). This is handled by recreateValidatorStatusIfNewValidatorsAreFound + // (post-Electra) + ValidatorStatuses validatorStatuses = validatorStatusFactory.createValidatorStatuses(preState); final UInt64 currentEpoch = beaconStateAccessors.getCurrentEpoch(state); final TotalBalances totalBalances = validatorStatuses.getTotalBalances(); @@ -133,6 +127,12 @@ protected void processEpoch(final BeaconState preState, final MutableBeaconState processSlashings(state, validatorStatuses); processEth1DataReset(state); processPendingDeposits(state); + + if (shouldCheckNewValidatorsDuringEpochProcessing()) { + validatorStatuses = + recreateValidatorStatusIfNewValidatorsAreFound(state, validatorStatuses, currentEpoch); + } + processPendingConsolidations(state); processEffectiveBalanceUpdates(state, validatorStatuses.getStatuses()); processSlashingsReset(state); @@ -148,6 +148,40 @@ protected void processEpoch(final BeaconState preState, final MutableBeaconState } } + @VisibleForTesting + public ValidatorStatuses recreateValidatorStatusIfNewValidatorsAreFound( + final BeaconState state, + final ValidatorStatuses validatorStatuses, + final UInt64 currentEpoch) { + final int cachedValidatorCount = validatorStatuses.getValidatorCount(); + final int stateValidatorCount = state.getValidators().size(); + if (stateValidatorCount > cachedValidatorCount) { + // New validators added, create new validator statuses + final List newValidatorStatuses = + new ArrayList<>(stateValidatorCount - cachedValidatorCount); + for (int i = cachedValidatorCount; i < stateValidatorCount; i++) { + final ValidatorStatus status = + validatorStatusFactory.createValidatorStatus( + state.getValidators().get(i), currentEpoch.minusMinZero(1), currentEpoch); + newValidatorStatuses.add(status); + } + return validatorStatusFactory.recreateValidatorStatuses( + validatorStatuses, newValidatorStatuses); + } else { + return validatorStatuses; + } + } + + /** + * This method is used to decide if we want to check the possibility of the validator set changing + * mid-processing an epoch. This is only required post-Electra. + * + * @return false by default, true post-Electra (EpochProcessorElectra overrides this method) + */ + protected boolean shouldCheckNewValidatorsDuringEpochProcessing() { + return false; + } + private void updateTransitionCaches( final MutableBeaconState state, final UInt64 currentEpoch, @@ -463,7 +497,7 @@ public void processEffectiveBalanceUpdates( final UInt64 maxEffectiveBalance = specConfig.getMaxEffectiveBalance(); final UInt64 hysteresisQuotient = specConfig.getHysteresisQuotient(); final UInt64 effectiveBalanceIncrement = specConfig.getEffectiveBalanceIncrement(); - for (int index = 0; index < validators.size(); index++) { + for (int index = 0; index < statuses.size(); index++) { final ValidatorStatus status = statuses.get(index); final UInt64 balance = balances.getElement(index); diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/common/statetransition/epoch/status/AbstractValidatorStatusFactory.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/common/statetransition/epoch/status/AbstractValidatorStatusFactory.java index cbf3d673201..18cceb43e60 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/common/statetransition/epoch/status/AbstractValidatorStatusFactory.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/common/statetransition/epoch/status/AbstractValidatorStatusFactory.java @@ -16,6 +16,7 @@ import static tech.pegasys.teku.infrastructure.unsigned.UInt64.MAX_VALUE; import java.util.List; +import java.util.stream.Stream; import org.apache.tuweni.bytes.Bytes32; import tech.pegasys.teku.infrastructure.ssz.SszList; import tech.pegasys.teku.infrastructure.unsigned.UInt64; @@ -70,6 +71,16 @@ public ValidatorStatuses createValidatorStatuses(final BeaconState state) { return new ValidatorStatuses(statuses, createTotalBalances(state, statuses)); } + @Override + public ValidatorStatuses recreateValidatorStatuses( + final ValidatorStatuses validatorStatuses, + final List validatorStatusesToAppend) { + final List validatorStatusesList = + Stream.concat(validatorStatuses.getStatuses().stream(), validatorStatusesToAppend.stream()) + .toList(); + return new ValidatorStatuses(validatorStatusesList, validatorStatuses.getTotalBalances()); + } + private TotalBalances createTotalBalances( final BeaconState state, final List statuses) { final TransitionCaches transitionCaches = BeaconStateCache.getTransitionCaches(state); diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/common/statetransition/epoch/status/ValidatorStatusFactory.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/common/statetransition/epoch/status/ValidatorStatusFactory.java index 9ebc9b02fe2..62f0cb3a031 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/common/statetransition/epoch/status/ValidatorStatusFactory.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/common/statetransition/epoch/status/ValidatorStatusFactory.java @@ -13,13 +13,28 @@ package tech.pegasys.teku.spec.logic.common.statetransition.epoch.status; +import java.util.List; import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.datastructures.state.Validator; import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconState; public interface ValidatorStatusFactory { + ValidatorStatuses createValidatorStatuses(BeaconState state); ValidatorStatus createValidatorStatus( final Validator validator, final UInt64 previousEpoch, final UInt64 currentEpoch); + + /** + * Creates a new ValidatorStatuses object with the existing list of statuses and the new statuses + * from the given list. This is cheaper than creating a new one using createValidatorStatus method + * because it will not recompute any data for validators already mapped in this object. + * + * @param validatorStatuses existing ValidatorStatuses object + * @param validatorStatusesToAppend new statuses to append to the exiting ValidatorStatuses object + * @return a new instance of ValidatorStatuses with both pre-existing and new validator statuses + */ + ValidatorStatuses recreateValidatorStatuses( + final ValidatorStatuses validatorStatuses, + final List validatorStatusesToAppend); } diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/electra/statetransition/epoch/EpochProcessorElectra.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/electra/statetransition/epoch/EpochProcessorElectra.java index ebd9a1edbc0..8eedf3e372e 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/electra/statetransition/epoch/EpochProcessorElectra.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/electra/statetransition/epoch/EpochProcessorElectra.java @@ -379,4 +379,9 @@ public void processSlashings( } } } + + @Override + protected boolean shouldCheckNewValidatorsDuringEpochProcessing() { + return true; + } } diff --git a/ethereum/spec/src/test/java/tech/pegasys/teku/spec/logic/common/statetransition/epoch/AbstractEpochProcessorTest.java b/ethereum/spec/src/test/java/tech/pegasys/teku/spec/logic/common/statetransition/epoch/AbstractEpochProcessorTest.java index 97f9dafe3c9..065554429b3 100644 --- a/ethereum/spec/src/test/java/tech/pegasys/teku/spec/logic/common/statetransition/epoch/AbstractEpochProcessorTest.java +++ b/ethereum/spec/src/test/java/tech/pegasys/teku/spec/logic/common/statetransition/epoch/AbstractEpochProcessorTest.java @@ -13,6 +13,7 @@ package tech.pegasys.teku.spec.logic.common.statetransition.epoch; +import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; @@ -23,6 +24,8 @@ import java.lang.reflect.Field; import java.time.Duration; +import java.util.List; +import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.logging.log4j.Logger; import org.junit.jupiter.api.Test; import org.junit.platform.commons.util.ReflectionUtils; @@ -36,6 +39,8 @@ import tech.pegasys.teku.spec.TestSpecFactory; import tech.pegasys.teku.spec.datastructures.state.Validator; import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconState; +import tech.pegasys.teku.spec.logic.common.statetransition.epoch.status.ValidatorStatusFactory; +import tech.pegasys.teku.spec.logic.common.statetransition.epoch.status.ValidatorStatuses; import tech.pegasys.teku.spec.logic.versions.capella.statetransition.epoch.EpochProcessorCapella; import tech.pegasys.teku.spec.util.DataStructureUtil; @@ -43,11 +48,7 @@ class AbstractEpochProcessorTest { private final Spec spec = TestSpecFactory.createMinimalCapella(); private final DataStructureUtil dataStructureUtil = new DataStructureUtil(spec); - private final StubTimeProvider timeProvider = StubTimeProvider.withTimeInSeconds(100_000L); - private final EpochProcessorCapella epochProcessor = - new EpochProcessorCapella( - (EpochProcessorCapella) spec.getGenesisSpec().getEpochProcessor(), timeProvider); - + private final EpochProcessor epochProcessor = spec.getGenesisSpec().getEpochProcessor(); private final int throttlingPeriod = 1; // expect maximum of one call per second private static final Logger LOGGER = mock(Logger.class); private final Throttler loggerThrottler = spyLogThrottler(LOGGER, throttlingPeriod); @@ -56,6 +57,10 @@ class AbstractEpochProcessorTest { @Test public void shouldThrottleInactivityLeakLogs() throws Exception { + final StubTimeProvider timeProvider = StubTimeProvider.withTimeInSeconds(100_000L); + final EpochProcessor epochProcessor = spec.getGenesisSpec().getEpochProcessor(); + FieldUtils.writeField(epochProcessor, "timeProvider", timeProvider, true); + // First two processEpoch calls within the same second epochProcessor.processEpoch(state); epochProcessor.processEpoch(advanceNSlots(state, 1)); @@ -111,4 +116,62 @@ private Throttler spyLogThrottler(final Logger logger, final int throttl return loggerThrottler; } + + @Test + public void shouldCheckNewValidatorsDuringEpochProcessingReturnsFalse() { + assertThat( + ((AbstractEpochProcessor) epochProcessor) + .shouldCheckNewValidatorsDuringEpochProcessing()) + .isFalse(); + } + + @Test + public void recreateValidatorStatusWithNoNewValidators() { + final BeaconState preState = + dataStructureUtil.stateBuilder(spec.getGenesisSpec().getMilestone(), 10, 3).build(); + final UInt64 currentEpoch = spec.computeEpochAtSlot(preState.getSlot()); + final ValidatorStatusFactory validatorStatusFactory = + spec.atSlot(preState.getSlot()).getValidatorStatusFactory(); + + final ValidatorStatuses validatorStatuses = + validatorStatusFactory.createValidatorStatuses(preState); + + final ValidatorStatuses newValidatorStatuses = + ((AbstractEpochProcessor) epochProcessor) + .recreateValidatorStatusIfNewValidatorsAreFound( + preState, validatorStatuses, currentEpoch); + + assertThat(preState.getValidators().size()) + .isEqualTo(newValidatorStatuses.getStatuses().size()); + } + + @Test + public void recreateValidatorStatusWithNewValidators() { + final BeaconState preState = + dataStructureUtil.stateBuilder(spec.getGenesisSpec().getMilestone(), 10, 3).build(); + final UInt64 currentEpoch = spec.computeEpochAtSlot(preState.getSlot()); + final ValidatorStatusFactory validatorStatusFactory = + spec.atSlot(preState.getSlot()).getValidatorStatusFactory(); + + final ValidatorStatuses validatorStatuses = + validatorStatusFactory.createValidatorStatuses(preState); + + final List newValidators = + List.of( + dataStructureUtil.randomValidator(), + dataStructureUtil.randomValidator(), + dataStructureUtil.randomValidator()); + final BeaconState postState = + preState.updated(state -> newValidators.forEach(state.getValidators()::append)); + + final ValidatorStatuses newValidatorStatuses = + ((AbstractEpochProcessor) epochProcessor) + .recreateValidatorStatusIfNewValidatorsAreFound( + postState, validatorStatuses, currentEpoch); + + assertThat(preState.getValidators().size() + newValidators.size()) + .isEqualTo(newValidatorStatuses.getStatuses().size()); + assertThat(postState.getValidators().size()) + .isEqualTo(newValidatorStatuses.getStatuses().size()); + } } diff --git a/ethereum/spec/src/test/java/tech/pegasys/teku/spec/logic/common/statetransition/epoch/status/AbstractValidatorStatusFactoryTest.java b/ethereum/spec/src/test/java/tech/pegasys/teku/spec/logic/common/statetransition/epoch/status/AbstractValidatorStatusFactoryTest.java index 3239295ab16..5a86cb8a833 100644 --- a/ethereum/spec/src/test/java/tech/pegasys/teku/spec/logic/common/statetransition/epoch/status/AbstractValidatorStatusFactoryTest.java +++ b/ethereum/spec/src/test/java/tech/pegasys/teku/spec/logic/common/statetransition/epoch/status/AbstractValidatorStatusFactoryTest.java @@ -15,10 +15,16 @@ import static java.util.Collections.emptyList; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static tech.pegasys.teku.spec.config.SpecConfig.FAR_FUTURE_EPOCH; import java.util.List; import java.util.stream.Stream; +import org.apache.commons.lang3.reflect.FieldUtils; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -28,6 +34,8 @@ import tech.pegasys.teku.spec.Spec; import tech.pegasys.teku.spec.config.SpecConfig; import tech.pegasys.teku.spec.datastructures.state.Validator; +import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconState; +import tech.pegasys.teku.spec.logic.common.util.AttestationUtil; import tech.pegasys.teku.spec.util.DataStructureUtil; public abstract class AbstractValidatorStatusFactoryTest { @@ -249,6 +257,67 @@ void createTotalBalances_shouldReturnMinimumOfOneEffectiveBalanceIncrement() { assertThat(balances.getPreviousEpochHeadAttesters()).isEqualTo(effectiveBalanceInc); } + @Test + public void recreateValidatorStatusesShouldAppendNewValidatorsAndKeepOrder() + throws IllegalAccessException { + final BeaconState state = + dataStructureUtil + .stateBuilder(spec.getGenesisSpec().getMilestone(), 3, 1) + .setSlotToStartOfEpoch(UInt64.ONE) + .build(); + // Magic to get around requiring a full state with valid previous and current epoch attestations + // (only on Phase0) + final ValidatorStatusFactory factory = createFactory(); + FieldUtils.writeField(factory, "attestationUtil", mock(AttestationUtil.class), true); + + final ValidatorStatuses validatorStatuses = factory.createValidatorStatuses(state); + + final UInt64 currentEpoch = spec.computeEpochAtSlot(state.getSlot()); + final Validator newValidator = dataStructureUtil.validatorBuilder().slashed(true).build(); + final ValidatorStatus newValidatorStatus = + factory.createValidatorStatus(newValidator, currentEpoch.minusMinZero(1), currentEpoch); + + final ValidatorStatuses updatedValidatorStatuses = + factory.recreateValidatorStatuses(validatorStatuses, List.of(newValidatorStatus)); + + final ValidatorStatus[] expectedStatuses = + Stream.concat(validatorStatuses.getStatuses().stream(), Stream.of(newValidatorStatus)) + .toArray(ValidatorStatus[]::new); + + assertThat(updatedValidatorStatuses.getStatuses()).containsExactly(expectedStatuses); + } + + @Test + public void shouldNotRecalculateValidatorStatusForPreviousExistingValidators() + throws IllegalAccessException { + final ValidatorStatusFactory factory = spy(createFactory()); + // Magic to get around requiring a full state with valid previous and current epoch attestations + // (only on Phase0) + FieldUtils.writeField(factory, "attestationUtil", mock(AttestationUtil.class), true); + + final int validatorCount = 10; + final BeaconState state = + dataStructureUtil + .stateBuilder(spec.getGenesisSpec().getMilestone(), validatorCount, 1) + .build(); + final UInt64 currentEpoch = spec.computeEpochAtSlot(state.getSlot()); + final ValidatorStatuses validatorStatuses = factory.createValidatorStatuses(state); + + // Created ValidatorStatus for all validators in state + verify(factory, times(validatorCount)).createValidatorStatus(any(), any(), any()); + + final Validator newValidator = dataStructureUtil.validatorBuilder().slashed(true).build(); + final ValidatorStatus newValidatorStatus = + factory.createValidatorStatus(newValidator, currentEpoch.minusMinZero(1), currentEpoch); + // Created ValidatorStatus for the new validator + verify(factory, times(validatorCount + 1)).createValidatorStatus(any(), any(), any()); + + factory.recreateValidatorStatuses(validatorStatuses, List.of(newValidatorStatus)); + + // Verifying that recreateValidatorStatuses does not trigger any ValidatorStatus creation + verify(factory, times(validatorCount + 1)).createValidatorStatus(any(), any(), any()); + } + private ValidatorStatus createValidator(final int effectiveBalance) { return new ValidatorStatus( false, false, balance(effectiveBalance), withdrawableEpoch, true, true, true); diff --git a/ethereum/spec/src/test/java/tech/pegasys/teku/spec/logic/versions/electra/statetransition/epoch/EpochProcessorElectraTest.java b/ethereum/spec/src/test/java/tech/pegasys/teku/spec/logic/versions/electra/statetransition/epoch/EpochProcessorElectraTest.java new file mode 100644 index 00000000000..0fa2d60d824 --- /dev/null +++ b/ethereum/spec/src/test/java/tech/pegasys/teku/spec/logic/versions/electra/statetransition/epoch/EpochProcessorElectraTest.java @@ -0,0 +1,32 @@ +/* + * Copyright Consensys Software Inc., 2024 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.spec.logic.versions.electra.statetransition.epoch; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; +import tech.pegasys.teku.spec.Spec; +import tech.pegasys.teku.spec.TestSpecFactory; + +class EpochProcessorElectraTest { + + private final Spec spec = TestSpecFactory.createMinimalElectra(); + private final EpochProcessorElectra epochProcessor = + (EpochProcessorElectra) spec.getGenesisSpec().getEpochProcessor(); + + @Test + public void shouldCheckNewValidatorsDuringEpochProcessingReturnsTrue() { + assertThat(epochProcessor.shouldCheckNewValidatorsDuringEpochProcessing()).isTrue(); + } +} From 0e86c08e04de9ee36b3d4cf24148208d7fc5d3c1 Mon Sep 17 00:00:00 2001 From: Mehdi AOUADI Date: Tue, 3 Dec 2024 16:08:27 +0100 Subject: [PATCH 06/20] move MAX_BLOBS_PER_BLOCK from preset to config params (#8879) * move MAX_BLOBS_PER_BLOCK from preset to config params --- .../resources/tech/pegasys/teku/spec/config/configs/chiado.yaml | 2 ++ .../tech/pegasys/teku/spec/config/configs/ephemery.yaml | 2 ++ .../resources/tech/pegasys/teku/spec/config/configs/gnosis.yaml | 2 ++ .../tech/pegasys/teku/spec/config/configs/less-swift.yaml | 2 ++ .../resources/tech/pegasys/teku/spec/config/configs/lukso.yaml | 2 ++ .../tech/pegasys/teku/spec/config/configs/mainnet.yaml | 2 ++ .../tech/pegasys/teku/spec/config/configs/minimal.yaml | 2 ++ .../tech/pegasys/teku/spec/config/configs/sepolia.yaml | 2 ++ .../resources/tech/pegasys/teku/spec/config/configs/swift.yaml | 2 ++ .../tech/pegasys/teku/spec/config/presets/gnosis/deneb.yaml | 2 -- .../tech/pegasys/teku/spec/config/presets/mainnet/deneb.yaml | 2 -- .../tech/pegasys/teku/spec/config/presets/minimal/deneb.yaml | 2 -- .../tech/pegasys/teku/spec/config/presets/swift/deneb.yaml | 2 -- .../teku/spec/config/standard/presets/mainnet/deneb.yaml | 2 -- .../teku/spec/config/standard/presets/minimal/deneb.yaml | 2 -- .../pegasys/teku/spec/config/standard/presets/swift/deneb.yaml | 2 -- .../tech/pegasys/teku/cli/subcommand/deneb_config.yaml | 2 ++ 17 files changed, 20 insertions(+), 14 deletions(-) diff --git a/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/configs/chiado.yaml b/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/configs/chiado.yaml index fda0da2f5b4..7ba59a19848 100644 --- a/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/configs/chiado.yaml +++ b/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/configs/chiado.yaml @@ -134,6 +134,8 @@ MAX_REQUEST_BLOB_SIDECARS: 768 MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 16384 # `6` BLOB_SIDECAR_SUBNET_COUNT: 6 +# `6` +MAX_BLOBS_PER_BLOCK: 6 # Electra MAX_BLOBS_PER_BLOCK_ELECTRA: 6 diff --git a/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/configs/ephemery.yaml b/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/configs/ephemery.yaml index 8df5977f605..0b3f87844d2 100644 --- a/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/configs/ephemery.yaml +++ b/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/configs/ephemery.yaml @@ -123,6 +123,8 @@ MAX_REQUEST_BLOB_SIDECARS: 768 MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 4096 # `6` BLOB_SIDECAR_SUBNET_COUNT: 6 +# `6` +MAX_BLOBS_PER_BLOCK: 6 # Electra MAX_BLOBS_PER_BLOCK_ELECTRA: 6 diff --git a/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/configs/gnosis.yaml b/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/configs/gnosis.yaml index 42b99807212..c7600b8648a 100644 --- a/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/configs/gnosis.yaml +++ b/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/configs/gnosis.yaml @@ -133,6 +133,8 @@ MAX_REQUEST_BLOB_SIDECARS: 768 MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 16384 # `6` BLOB_SIDECAR_SUBNET_COUNT: 6 +# `6` +MAX_BLOBS_PER_BLOCK: 6 # Electra MAX_BLOBS_PER_BLOCK_ELECTRA: 6 diff --git a/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/configs/less-swift.yaml b/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/configs/less-swift.yaml index 63cb669375b..6adebb9c009 100644 --- a/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/configs/less-swift.yaml +++ b/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/configs/less-swift.yaml @@ -136,6 +136,8 @@ MAX_REQUEST_BLOB_SIDECARS: 768 MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 4096 # `6` BLOB_SIDECAR_SUBNET_COUNT: 6 +# `6` +MAX_BLOBS_PER_BLOCK: 6 # Electra MAX_BLOBS_PER_BLOCK_ELECTRA: 6 diff --git a/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/configs/lukso.yaml b/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/configs/lukso.yaml index 578d711cccc..47ad8fa06a4 100644 --- a/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/configs/lukso.yaml +++ b/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/configs/lukso.yaml @@ -155,6 +155,8 @@ MAX_REQUEST_BLOB_SIDECARS: 768 MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 4096 # `6` BLOB_SIDECAR_SUBNET_COUNT: 6 +# `6` +MAX_BLOBS_PER_BLOCK: 6 # Electra MAX_BLOBS_PER_BLOCK_ELECTRA: 6 diff --git a/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/configs/mainnet.yaml b/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/configs/mainnet.yaml index 18ca69b53e4..3c831ba06eb 100644 --- a/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/configs/mainnet.yaml +++ b/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/configs/mainnet.yaml @@ -142,6 +142,8 @@ MAX_REQUEST_BLOB_SIDECARS: 768 MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 4096 # `6` BLOB_SIDECAR_SUBNET_COUNT: 6 +# `6` +MAX_BLOBS_PER_BLOCK: 6 # Electra MAX_BLOBS_PER_BLOCK_ELECTRA: 6 diff --git a/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/configs/minimal.yaml b/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/configs/minimal.yaml index f3386e056f3..1a6ea3507fa 100644 --- a/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/configs/minimal.yaml +++ b/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/configs/minimal.yaml @@ -142,6 +142,8 @@ MAX_REQUEST_BLOB_SIDECARS: 768 MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 4096 # `6` BLOB_SIDECAR_SUBNET_COUNT: 6 +# `6` +MAX_BLOBS_PER_BLOCK: 6 # Electra MAX_BLOBS_PER_BLOCK_ELECTRA: 6 diff --git a/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/configs/sepolia.yaml b/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/configs/sepolia.yaml index f4eac63a5b5..f00e64693b0 100644 --- a/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/configs/sepolia.yaml +++ b/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/configs/sepolia.yaml @@ -118,6 +118,8 @@ MAX_REQUEST_BLOB_SIDECARS: 768 MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 4096 # `6` BLOB_SIDECAR_SUBNET_COUNT: 6 +# `6` +MAX_BLOBS_PER_BLOCK: 6 # Electra MAX_BLOBS_PER_BLOCK_ELECTRA: 6 diff --git a/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/configs/swift.yaml b/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/configs/swift.yaml index 72afef1833e..4dd4e891242 100644 --- a/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/configs/swift.yaml +++ b/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/configs/swift.yaml @@ -137,6 +137,8 @@ MAX_REQUEST_BLOB_SIDECARS: 768 MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 4096 # `6` BLOB_SIDECAR_SUBNET_COUNT: 6 +# `6` +MAX_BLOBS_PER_BLOCK: 6 # Electra MAX_BLOBS_PER_BLOCK_ELECTRA: 6 diff --git a/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/presets/gnosis/deneb.yaml b/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/presets/gnosis/deneb.yaml index e2fb3e76260..2a9fb62d70d 100644 --- a/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/presets/gnosis/deneb.yaml +++ b/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/presets/gnosis/deneb.yaml @@ -6,7 +6,5 @@ FIELD_ELEMENTS_PER_BLOB: 4096 # `uint64(2**12)` (= 4096) MAX_BLOB_COMMITMENTS_PER_BLOCK: 4096 -# `uint64(6)` -MAX_BLOBS_PER_BLOCK: 6 # `floorlog2(get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments')) + 1 + ceillog2(MAX_BLOB_COMMITMENTS_PER_BLOCK)` = 4 + 1 + 12 = 17 KZG_COMMITMENT_INCLUSION_PROOF_DEPTH: 17 \ No newline at end of file diff --git a/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/presets/mainnet/deneb.yaml b/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/presets/mainnet/deneb.yaml index 8dfd353076c..76bab10c380 100644 --- a/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/presets/mainnet/deneb.yaml +++ b/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/presets/mainnet/deneb.yaml @@ -6,7 +6,5 @@ FIELD_ELEMENTS_PER_BLOB: 4096 # `uint64(2**12)` (= 4096) MAX_BLOB_COMMITMENTS_PER_BLOCK: 4096 -# `uint64(6)` -MAX_BLOBS_PER_BLOCK: 6 # `floorlog2(get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments')) + 1 + ceillog2(MAX_BLOB_COMMITMENTS_PER_BLOCK)` = 4 + 1 + 12 = 17 KZG_COMMITMENT_INCLUSION_PROOF_DEPTH: 17 \ No newline at end of file diff --git a/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/presets/minimal/deneb.yaml b/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/presets/minimal/deneb.yaml index 03d2cd3bc05..f71173db736 100644 --- a/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/presets/minimal/deneb.yaml +++ b/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/presets/minimal/deneb.yaml @@ -6,7 +6,5 @@ FIELD_ELEMENTS_PER_BLOB: 4096 # [customized] MAX_BLOB_COMMITMENTS_PER_BLOCK: 16 -# `uint64(6)` -MAX_BLOBS_PER_BLOCK: 6 # [customized] `floorlog2(get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments')) + 1 + ceillog2(MAX_BLOB_COMMITMENTS_PER_BLOCK)` = 4 + 1 + 4 = 9 KZG_COMMITMENT_INCLUSION_PROOF_DEPTH: 9 \ No newline at end of file diff --git a/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/presets/swift/deneb.yaml b/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/presets/swift/deneb.yaml index 03d2cd3bc05..f71173db736 100644 --- a/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/presets/swift/deneb.yaml +++ b/ethereum/spec/src/main/resources/tech/pegasys/teku/spec/config/presets/swift/deneb.yaml @@ -6,7 +6,5 @@ FIELD_ELEMENTS_PER_BLOB: 4096 # [customized] MAX_BLOB_COMMITMENTS_PER_BLOCK: 16 -# `uint64(6)` -MAX_BLOBS_PER_BLOCK: 6 # [customized] `floorlog2(get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments')) + 1 + ceillog2(MAX_BLOB_COMMITMENTS_PER_BLOCK)` = 4 + 1 + 4 = 9 KZG_COMMITMENT_INCLUSION_PROOF_DEPTH: 9 \ No newline at end of file diff --git a/ethereum/spec/src/test/resources/tech/pegasys/teku/spec/config/standard/presets/mainnet/deneb.yaml b/ethereum/spec/src/test/resources/tech/pegasys/teku/spec/config/standard/presets/mainnet/deneb.yaml index 8dfd353076c..76bab10c380 100644 --- a/ethereum/spec/src/test/resources/tech/pegasys/teku/spec/config/standard/presets/mainnet/deneb.yaml +++ b/ethereum/spec/src/test/resources/tech/pegasys/teku/spec/config/standard/presets/mainnet/deneb.yaml @@ -6,7 +6,5 @@ FIELD_ELEMENTS_PER_BLOB: 4096 # `uint64(2**12)` (= 4096) MAX_BLOB_COMMITMENTS_PER_BLOCK: 4096 -# `uint64(6)` -MAX_BLOBS_PER_BLOCK: 6 # `floorlog2(get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments')) + 1 + ceillog2(MAX_BLOB_COMMITMENTS_PER_BLOCK)` = 4 + 1 + 12 = 17 KZG_COMMITMENT_INCLUSION_PROOF_DEPTH: 17 \ No newline at end of file diff --git a/ethereum/spec/src/test/resources/tech/pegasys/teku/spec/config/standard/presets/minimal/deneb.yaml b/ethereum/spec/src/test/resources/tech/pegasys/teku/spec/config/standard/presets/minimal/deneb.yaml index 03d2cd3bc05..f71173db736 100644 --- a/ethereum/spec/src/test/resources/tech/pegasys/teku/spec/config/standard/presets/minimal/deneb.yaml +++ b/ethereum/spec/src/test/resources/tech/pegasys/teku/spec/config/standard/presets/minimal/deneb.yaml @@ -6,7 +6,5 @@ FIELD_ELEMENTS_PER_BLOB: 4096 # [customized] MAX_BLOB_COMMITMENTS_PER_BLOCK: 16 -# `uint64(6)` -MAX_BLOBS_PER_BLOCK: 6 # [customized] `floorlog2(get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments')) + 1 + ceillog2(MAX_BLOB_COMMITMENTS_PER_BLOCK)` = 4 + 1 + 4 = 9 KZG_COMMITMENT_INCLUSION_PROOF_DEPTH: 9 \ No newline at end of file diff --git a/ethereum/spec/src/test/resources/tech/pegasys/teku/spec/config/standard/presets/swift/deneb.yaml b/ethereum/spec/src/test/resources/tech/pegasys/teku/spec/config/standard/presets/swift/deneb.yaml index 03d2cd3bc05..f71173db736 100644 --- a/ethereum/spec/src/test/resources/tech/pegasys/teku/spec/config/standard/presets/swift/deneb.yaml +++ b/ethereum/spec/src/test/resources/tech/pegasys/teku/spec/config/standard/presets/swift/deneb.yaml @@ -6,7 +6,5 @@ FIELD_ELEMENTS_PER_BLOB: 4096 # [customized] MAX_BLOB_COMMITMENTS_PER_BLOCK: 16 -# `uint64(6)` -MAX_BLOBS_PER_BLOCK: 6 # [customized] `floorlog2(get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments')) + 1 + ceillog2(MAX_BLOB_COMMITMENTS_PER_BLOCK)` = 4 + 1 + 4 = 9 KZG_COMMITMENT_INCLUSION_PROOF_DEPTH: 9 \ No newline at end of file diff --git a/teku/src/test/resources/tech/pegasys/teku/cli/subcommand/deneb_config.yaml b/teku/src/test/resources/tech/pegasys/teku/cli/subcommand/deneb_config.yaml index 9a15730dcdc..49c8d3b11bd 100644 --- a/teku/src/test/resources/tech/pegasys/teku/cli/subcommand/deneb_config.yaml +++ b/teku/src/test/resources/tech/pegasys/teku/cli/subcommand/deneb_config.yaml @@ -139,6 +139,8 @@ MAX_REQUEST_BLOB_SIDECARS: 768 MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 4096 # `6` BLOB_SIDECAR_SUBNET_COUNT: 6 +# `6` +MAX_BLOBS_PER_BLOCK: 6 # Electra MAX_BLOBS_PER_BLOCK_ELECTRA: 6 From c0263cad947c049de926b09a60d348284867b625 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Tue, 3 Dec 2024 22:19:23 +0400 Subject: [PATCH 07/20] Don't subscribe in AttestationTopicSubscriber if subscription is outdated (#8837) --- .../subnets/AttestationTopicSubscriber.java | 30 ++++++++++++-- .../AttestationTopicSubscriberTest.java | 41 +++++++++++++++++++ 2 files changed, 67 insertions(+), 4 deletions(-) diff --git a/networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/gossip/subnets/AttestationTopicSubscriber.java b/networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/gossip/subnets/AttestationTopicSubscriber.java index ccf785f92c8..250902a79f6 100644 --- a/networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/gossip/subnets/AttestationTopicSubscriber.java +++ b/networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/gossip/subnets/AttestationTopicSubscriber.java @@ -21,6 +21,7 @@ import it.unimi.dsi.fastutil.ints.IntSet; import java.util.Iterator; import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import tech.pegasys.teku.ethereum.events.SlotEventsChannel; @@ -39,6 +40,7 @@ public class AttestationTopicSubscriber implements SlotEventsChannel { private final Eth2P2PNetwork eth2P2PNetwork; private final Spec spec; private final SettableLabelledGauge subnetSubscriptionsGauge; + private final AtomicReference currentSlot = new AtomicReference<>(null); public AttestationTopicSubscriber( final Spec spec, @@ -56,6 +58,15 @@ public synchronized void subscribeToCommitteeForAggregation( aggregationSlot, UInt64.valueOf(committeeIndex), committeesAtSlot); final UInt64 currentUnsubscriptionSlot = subnetIdToUnsubscribeSlot.getOrDefault(subnetId, ZERO); final UInt64 unsubscribeSlot = currentUnsubscriptionSlot.max(aggregationSlot); + final UInt64 maybeCurrentSlot = currentSlot.get(); + if (maybeCurrentSlot != null && unsubscribeSlot.isLessThan(maybeCurrentSlot)) { + LOG.trace( + "Skipping outdated aggregation subnet {} with unsubscribe due at slot {}", + subnetId, + unsubscribeSlot); + return; + } + if (currentUnsubscriptionSlot.equals(ZERO)) { eth2P2PNetwork.subscribeToAttestationSubnetId(subnetId); toggleAggregateSubscriptionMetric(subnetId, false); @@ -96,15 +107,25 @@ public synchronized void subscribeToPersistentSubnets( boolean shouldUpdateENR = false; for (SubnetSubscription subnetSubscription : newSubscriptions) { - int subnetId = subnetSubscription.subnetId(); + final int subnetId = subnetSubscription.subnetId(); + final UInt64 maybeCurrentSlot = currentSlot.get(); + if (maybeCurrentSlot != null + && subnetSubscription.unsubscriptionSlot().isLessThan(maybeCurrentSlot)) { + LOG.trace( + "Skipping outdated persistent subnet {} with unsubscribe due at slot {}", + subnetId, + subnetSubscription.unsubscriptionSlot()); + continue; + } + shouldUpdateENR = persistentSubnetIdSet.add(subnetId) || shouldUpdateENR; LOG.trace( "Subscribing to persistent subnet {} with unsubscribe due at slot {}", subnetId, subnetSubscription.unsubscriptionSlot()); if (subnetIdToUnsubscribeSlot.containsKey(subnetId)) { - UInt64 existingUnsubscriptionSlot = subnetIdToUnsubscribeSlot.get(subnetId); - UInt64 unsubscriptionSlot = + final UInt64 existingUnsubscriptionSlot = subnetIdToUnsubscribeSlot.get(subnetId); + final UInt64 unsubscriptionSlot = existingUnsubscriptionSlot.max(subnetSubscription.unsubscriptionSlot()); LOG.trace( "Already subscribed to subnet {}, updating unsubscription slot to {}", @@ -127,6 +148,7 @@ public synchronized void subscribeToPersistentSubnets( @Override public synchronized void onSlot(final UInt64 slot) { + currentSlot.set(slot); boolean shouldUpdateENR = false; final Iterator> iterator = @@ -134,7 +156,7 @@ public synchronized void onSlot(final UInt64 slot) { while (iterator.hasNext()) { final Int2ObjectMap.Entry entry = iterator.next(); if (entry.getValue().compareTo(slot) < 0) { - int subnetId = entry.getIntKey(); + final int subnetId = entry.getIntKey(); LOG.trace("Unsubscribing from subnet {}", subnetId); eth2P2PNetwork.unsubscribeFromAttestationSubnetId(subnetId); if (persistentSubnetIdSet.contains(subnetId)) { diff --git a/networking/eth2/src/test/java/tech/pegasys/teku/networking/eth2/gossip/subnets/AttestationTopicSubscriberTest.java b/networking/eth2/src/test/java/tech/pegasys/teku/networking/eth2/gossip/subnets/AttestationTopicSubscriberTest.java index 4ff2241e165..5fbdc1e466a 100644 --- a/networking/eth2/src/test/java/tech/pegasys/teku/networking/eth2/gossip/subnets/AttestationTopicSubscriberTest.java +++ b/networking/eth2/src/test/java/tech/pegasys/teku/networking/eth2/gossip/subnets/AttestationTopicSubscriberTest.java @@ -15,6 +15,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; @@ -47,6 +48,7 @@ public void shouldSubscribeToSubnet() { final int committeeId = 10; final int subnetId = spec.computeSubnetForCommittee(ONE, UInt64.valueOf(committeeId), COMMITTEES_AT_SLOT); + subscriber.onSlot(ONE); subscriber.subscribeToCommitteeForAggregation(committeeId, COMMITTEES_AT_SLOT, ONE); verify(settableLabelledGaugeMock) @@ -155,6 +157,44 @@ public void shouldPreserveLaterSubscriptionPeriodWhenEarlierSlotAdded() { verify(eth2P2PNetwork).unsubscribeFromAttestationSubnetId(subnetId); } + @Test + public void shouldNotSubscribeForExpiredAggregationSubnet() { + final int committeeId = 3; + final UInt64 slot = UInt64.valueOf(10); + final int subnetId = + spec.computeSubnetForCommittee(slot, UInt64.valueOf(committeeId), COMMITTEES_AT_SLOT); + // Sanity check second subscription is for the same subnet ID. + assertThat(subnetId) + .isEqualTo( + spec.computeSubnetForCommittee(slot, UInt64.valueOf(committeeId), COMMITTEES_AT_SLOT)); + + subscriber.onSlot(slot.plus(ONE)); + subscriber.subscribeToCommitteeForAggregation(committeeId, COMMITTEES_AT_SLOT, slot); + verifyNoMoreInteractions(settableLabelledGaugeMock); + verify(eth2P2PNetwork, never()).subscribeToAttestationSubnetId(anyInt()); + verify(eth2P2PNetwork, never()).unsubscribeFromAttestationSubnetId(anyInt()); + } + + @Test + public void shouldNotSubscribeForExpiredPersistentSubnet() { + Set subnetSubscriptions = + Set.of( + new SubnetSubscription(2, UInt64.valueOf(15)), + new SubnetSubscription(1, UInt64.valueOf(20))); + + subscriber.onSlot(UInt64.valueOf(16)); + subscriber.subscribeToPersistentSubnets(subnetSubscriptions); + + verify(settableLabelledGaugeMock) + .set(1, String.format(AttestationTopicSubscriber.GAUGE_PERSISTENT_SUBNETS_LABEL, 1)); + verifyNoMoreInteractions(settableLabelledGaugeMock); + + verify(eth2P2PNetwork).setLongTermAttestationSubnetSubscriptions(IntSet.of(1)); + + verify(eth2P2PNetwork).subscribeToAttestationSubnetId(1); + verify(eth2P2PNetwork, never()).subscribeToAttestationSubnetId(eq(2)); + } + @Test public void shouldSubscribeToNewSubnetsAndUpdateENR_forPersistentSubscriptions() { Set subnetSubscriptions = @@ -162,6 +202,7 @@ public void shouldSubscribeToNewSubnetsAndUpdateENR_forPersistentSubscriptions() new SubnetSubscription(1, UInt64.valueOf(20)), new SubnetSubscription(2, UInt64.valueOf(15))); + subscriber.onSlot(UInt64.valueOf(15)); subscriber.subscribeToPersistentSubnets(subnetSubscriptions); verify(settableLabelledGaugeMock) From 88d23bc9593b132784f5a5a8644d88fbe4ca5ff4 Mon Sep 17 00:00:00 2001 From: Lucas Saldanha Date: Wed, 4 Dec 2024 12:07:22 +1300 Subject: [PATCH 08/20] Fix ExecutionRequestsDataCodec handling of missing request data (#8880) --- .../electra/ExecutionRequestsDataCodec.java | 6 ++++ .../ExecutionRequestsDataCodecTest.java | 29 +++++++++++++++++-- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/execution/versions/electra/ExecutionRequestsDataCodec.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/execution/versions/electra/ExecutionRequestsDataCodec.java index 82a78cacff2..de9266d880c 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/execution/versions/electra/ExecutionRequestsDataCodec.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/execution/versions/electra/ExecutionRequestsDataCodec.java @@ -44,12 +44,18 @@ public ExecutionRequests decode(final List executionRequests) { if (request.isEmpty()) { throw new IllegalArgumentException("Execution request data must not be empty"); } + final byte requestType = request.get(0); if (requestType <= previousRequestType) { throw new IllegalArgumentException( "Execution requests are not in strictly ascending order"); } + final Bytes requestData = request.slice(1); + if (requestData.isEmpty()) { + throw new IllegalArgumentException("Empty data for request type " + requestType); + } + switch (requestType) { case DepositRequest.REQUEST_TYPE -> executionRequestsBuilder.deposits( diff --git a/ethereum/spec/src/test/java/tech/pegasys/teku/spec/datastructures/execution/versions/electra/ExecutionRequestsDataCodecTest.java b/ethereum/spec/src/test/java/tech/pegasys/teku/spec/datastructures/execution/versions/electra/ExecutionRequestsDataCodecTest.java index 70b5e0e0aba..0aa99d83bc0 100644 --- a/ethereum/spec/src/test/java/tech/pegasys/teku/spec/datastructures/execution/versions/electra/ExecutionRequestsDataCodecTest.java +++ b/ethereum/spec/src/test/java/tech/pegasys/teku/spec/datastructures/execution/versions/electra/ExecutionRequestsDataCodecTest.java @@ -92,12 +92,14 @@ public void decodeExecutionRequestsDataWithOneRequestMissing() { @Test public void decodeExecutionRequestsDataWithInvalidRequestType() { + final Bytes invalidRequestType = Bytes.of(9); + final Bytes invalidTypeEncodedList = Bytes.concatenate(invalidRequestType, Bytes.random(10)); final List invalidExecutionRequestsData = - List.of(depositRequestListEncoded, withdrawalRequestsListEncoded, Bytes.of(9)); + List.of(depositRequestListEncoded, withdrawalRequestsListEncoded, invalidTypeEncodedList); assertThatThrownBy(() -> codec.decode(invalidExecutionRequestsData)) .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Invalid execution request type: 9"); + .hasMessage("Invalid execution request type: " + invalidRequestType.toInt()); } @Test @@ -126,6 +128,29 @@ public void decodeExecutionRequestDataWithRepeatedRequestsOfSameType() { .hasMessage("Execution requests are not in strictly ascending order"); } + @Test + public void decodeExecutionRequestsDataWithEmptyRequestData() { + // Element containing only the type by but no data + final List invalidEmptyRequestsData = List.of(DepositRequest.REQUEST_TYPE_PREFIX); + + assertThatThrownBy(() -> codec.decode(invalidEmptyRequestsData)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Empty data for request type 0"); + } + + @Test + public void decodeExecutionRequestsDataWithOneInvalidEmptyRequestData() { + final List invalidExecutionRequestsData = + List.of( + depositRequestListEncoded, + withdrawalRequestsListEncoded, + ConsolidationRequest.REQUEST_TYPE_PREFIX); + + assertThatThrownBy(() -> codec.decode(invalidExecutionRequestsData)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Empty data for request type 2"); + } + @Test public void encodeExecutionRequests() { final ExecutionRequests executionRequests = From 89bdee9d151fe27df62be6c42ac77f328d3752a5 Mon Sep 17 00:00:00 2001 From: Lucas Saldanha Date: Wed, 4 Dec 2024 13:38:32 +1300 Subject: [PATCH 09/20] Added SingleAttestation to ref-test ssz executor (#8881) --- .../teku/reference/phase0/ssz_static/SszTestExecutor.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/phase0/ssz_static/SszTestExecutor.java b/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/phase0/ssz_static/SszTestExecutor.java index e2b515afb3c..040a0bb926f 100644 --- a/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/phase0/ssz_static/SszTestExecutor.java +++ b/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/phase0/ssz_static/SszTestExecutor.java @@ -220,6 +220,11 @@ public class SszTestExecutor implements TestExecutor { schemas -> SchemaDefinitionsElectra.required(schemas) .getPendingPartialWithdrawalSchema())) + .put( + "ssz_static/SingleAttestation", + new SszTestExecutor<>( + schemas -> + SchemaDefinitionsElectra.required(schemas).getSingleAttestationSchema())) // Legacy Schemas (Not yet migrated to SchemaDefinitions) .put( From 9e80efa250c30cac175d975950a1056dc6ff54c8 Mon Sep 17 00:00:00 2001 From: Paul Harris Date: Wed, 4 Dec 2024 11:37:23 +1000 Subject: [PATCH 10/20] Added SingleAttestationEvent (#8873) - if the attestation arriving is a singleAttestation, will fire a single_attestation event, otherwise will trigger an attestation event. - at the moment we only see attestations, but added unit test and can re-verify once we have single attestations being produced... - can essentially listen to both by : `curl -H "Accept:text/event-stream" -N http://localhost:5051/eth/v1/events\?topics\=attestation,single_attestation` Signed-off-by: Paul Harris --- CHANGELOG.md | 5 ++-- .../beacon/paths/_eth_v1_events.json | 2 +- .../beaconrestapi/BeaconRestApiTypes.java | 10 ++++++-- .../v1/events/EventSubscriptionManager.java | 10 ++++++-- .../v1/events/SingleAttestationEvent.java | 23 +++++++++++++++++++ .../events/EventSubscriptionManagerTest.java | 20 ++++++++++++++-- .../teku/api/response/v1/EventType.java | 3 ++- 7 files changed, 63 insertions(+), 10 deletions(-) create mode 100644 data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/SingleAttestationEvent.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 226335a30bc..d868630e8f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,9 @@ ### Breaking Changes ### Additions and Improvements -- improve block publishing performance, especially relevant with locally produced blocks -- delay blobs publishing until the block is published to at least 1 peer, especially relevant with locally produced blocks with low upload bandwidth connections. Can be disabled via `--Xp2p-gossip-blobs-after-block-enabled=false` +- Improved block publishing performance, especially relevant with locally produced blocks +- Delayed blob publishing until the block is published to at least 1 peer, especially relevant with locally produced blocks with low upload bandwidth connections. Can be disabled via `--Xp2p-gossip-blobs-after-block-enabled=false` +- Added single_attestation event type for electra attestations ### Bug Fixes - Added a startup script for unix systems to ensure that when jemalloc is installed the script sets the LD_PRELOAD environment variable to the use the jemalloc library diff --git a/data/beaconrestapi/src/integration-test/resources/tech/pegasys/teku/beaconrestapi/beacon/paths/_eth_v1_events.json b/data/beaconrestapi/src/integration-test/resources/tech/pegasys/teku/beaconrestapi/beacon/paths/_eth_v1_events.json index 5a007f68729..c3806e52186 100644 --- a/data/beaconrestapi/src/integration-test/resources/tech/pegasys/teku/beaconrestapi/beacon/paths/_eth_v1_events.json +++ b/data/beaconrestapi/src/integration-test/resources/tech/pegasys/teku/beaconrestapi/beacon/paths/_eth_v1_events.json @@ -9,7 +9,7 @@ "in" : "query", "schema" : { "type" : "string", - "description" : "Event types to subscribe to. Available values include: [`head`, `finalized_checkpoint`, `chain_reorg`, `block`, `attestation`, `voluntary_exit`, `contribution_and_proof`, `blob_sidecar`]\n\n", + "description" : "Event types to subscribe to. Supported event types: [`attestation`, `attester_slashing`, `blob_sidecar`, `block_gossip`, `block`, `bls_to_execution_change`, `chain_reorg`, `contribution_and_proof`, `finalized_checkpoint`, `head`, `payload_attributes`, `proposer_slashing`, `single_attestation`, `sync_state`, `voluntary_exit`", "example" : "head" } } ], diff --git a/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/BeaconRestApiTypes.java b/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/BeaconRestApiTypes.java index c6b3b68f9a4..d136dc62da7 100644 --- a/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/BeaconRestApiTypes.java +++ b/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/BeaconRestApiTypes.java @@ -61,9 +61,12 @@ import static tech.pegasys.teku.infrastructure.json.types.CoreTypes.STRING_TYPE; import static tech.pegasys.teku.infrastructure.json.types.CoreTypes.UINT64_TYPE; +import java.util.EnumSet; import java.util.Locale; import java.util.function.Function; +import java.util.stream.Collectors; import org.apache.tuweni.bytes.Bytes32; +import tech.pegasys.teku.api.response.v1.EventType; import tech.pegasys.teku.bls.BLSSignature; import tech.pegasys.teku.ethereum.json.types.beacon.StatusParameter; import tech.pegasys.teku.infrastructure.http.RestApiConstants; @@ -208,8 +211,11 @@ public class BeaconRestApiTypes { TOPICS, CoreTypes.string( "Event types to subscribe to." - + " Available values include: [`head`, `finalized_checkpoint`, `chain_reorg`, `block`, " - + "`attestation`, `voluntary_exit`, `contribution_and_proof`, `blob_sidecar`]\n\n", + + " Supported event types: [" + + EnumSet.allOf(EventType.class).stream() + .map(val -> "`" + val.toString() + "`") + .sorted() + .collect(Collectors.joining(", ")), "head")); public static final SerializableTypeDefinition ROOT_TYPE = diff --git a/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java b/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java index 5fd4833b78e..fb0c4238425 100644 --- a/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java +++ b/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java @@ -211,8 +211,14 @@ protected void onSyncCommitteeContribution( } protected void onNewAttestation(final ValidatableAttestation attestation) { - final AttestationEvent attestationEvent = new AttestationEvent(attestation.getAttestation()); - notifySubscribersOfEvent(EventType.attestation, attestationEvent); + if (!attestation.getAttestation().isSingleAttestation()) { + final AttestationEvent attestationEvent = new AttestationEvent(attestation.getAttestation()); + notifySubscribersOfEvent(EventType.attestation, attestationEvent); + } else { + final SingleAttestationEvent attestationEvent = + new SingleAttestationEvent(attestation.getAttestation().toSingleAttestationRequired()); + notifySubscribersOfEvent(EventType.single_attestation, attestationEvent); + } } protected void onNewBlock(final SignedBeaconBlock block, final boolean executionOptimistic) { diff --git a/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/SingleAttestationEvent.java b/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/SingleAttestationEvent.java new file mode 100644 index 00000000000..f92b477cd94 --- /dev/null +++ b/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/SingleAttestationEvent.java @@ -0,0 +1,23 @@ +/* + * Copyright Consensys Software Inc., 2024 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.beaconrestapi.handlers.v1.events; + +import tech.pegasys.teku.spec.datastructures.operations.SingleAttestation; + +public class SingleAttestationEvent extends Event { + SingleAttestationEvent(final SingleAttestation attestation) { + super( + attestation.toSingleAttestationRequired().getSchema().getJsonTypeDefinition(), attestation); + } +} diff --git a/data/beaconrestapi/src/test/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManagerTest.java b/data/beaconrestapi/src/test/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManagerTest.java index 5b251fe5b74..2ce52de32d0 100644 --- a/data/beaconrestapi/src/test/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManagerTest.java +++ b/data/beaconrestapi/src/test/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManagerTest.java @@ -55,6 +55,7 @@ import tech.pegasys.teku.spec.datastructures.operations.ProposerSlashing; import tech.pegasys.teku.spec.datastructures.operations.SignedBlsToExecutionChange; import tech.pegasys.teku.spec.datastructures.operations.SignedVoluntaryExit; +import tech.pegasys.teku.spec.datastructures.operations.SingleAttestation; import tech.pegasys.teku.spec.datastructures.operations.versions.altair.SignedContributionAndProof; import tech.pegasys.teku.spec.datastructures.state.Checkpoint; import tech.pegasys.teku.spec.executionlayer.ForkChoiceState; @@ -65,7 +66,7 @@ import tech.pegasys.teku.storage.api.ReorgContext; public class EventSubscriptionManagerTest { - private final Spec spec = TestSpecFactory.createMinimalDeneb(); + private final Spec spec = TestSpecFactory.createMinimalElectra(); private final SpecConfig specConfig = spec.getGenesisSpecConfig(); private final DataStructureUtil data = new DataStructureUtil(spec); protected final NodeDataProvider nodeDataProvider = mock(NodeDataProvider.class); @@ -107,6 +108,7 @@ public class EventSubscriptionManagerTest { SignedBeaconBlock.create(data.randomSignedBeaconBlock(0)); private final BlobSidecar sampleBlobSidecar = data.randomBlobSidecar(); private final Attestation sampleAttestation = data.randomAttestation(0); + private final SingleAttestation singleAttestation = data.randomSingleAttestation(); private final AttesterSlashing sampleAttesterSlashing = spec.getGenesisSchemaDefinitions() @@ -123,7 +125,7 @@ public class EventSubscriptionManagerTest { data.randomPayloadBuildingAttributes(true); final PayloadAttributesData samplePayloadAttributesData = new PayloadAttributesData( - SpecMilestone.DENEB, + SpecMilestone.ELECTRA, new Data( samplePayloadAttributes.getProposalSlot(), samplePayloadAttributes.getParentBeaconBlockRoot(), @@ -299,6 +301,15 @@ void shouldPropagateAttestation() throws IOException { checkEvent("attestation", new AttestationEvent(sampleAttestation)); } + @Test + void shouldPropagateSingleAttestation() throws IOException { + when(req.getQueryString()).thenReturn("&topics=single_attestation"); + manager.registerClient(client1); + + triggerSingleAttestationEvent(); + checkEvent("single_attestation", new SingleAttestationEvent(singleAttestation)); + } + @Test void shouldPropagateAttesterSlashing() throws IOException { when(req.getQueryString()).thenReturn("&topics=attester_slashing"); @@ -438,6 +449,11 @@ private void triggerAttestationEvent() { asyncRunner.executeQueuedActions(); } + private void triggerSingleAttestationEvent() { + manager.onNewAttestation(ValidatableAttestation.from(spec, singleAttestation)); + asyncRunner.executeQueuedActions(); + } + private void triggerAttesterSlashingEvent() { manager.onNewAttesterSlashing(sampleAttesterSlashing, InternalValidationResult.ACCEPT, false); asyncRunner.executeQueuedActions(); diff --git a/data/serializer/src/main/java/tech/pegasys/teku/api/response/v1/EventType.java b/data/serializer/src/main/java/tech/pegasys/teku/api/response/v1/EventType.java index 95e42fff52f..e5b9863dd21 100644 --- a/data/serializer/src/main/java/tech/pegasys/teku/api/response/v1/EventType.java +++ b/data/serializer/src/main/java/tech/pegasys/teku/api/response/v1/EventType.java @@ -30,7 +30,8 @@ public enum EventType { attester_slashing, proposer_slashing, payload_attributes, - block_gossip; + block_gossip, + single_attestation; public static List getTopics(final List topics) { return topics.stream().map(EventType::valueOf).toList(); From 7f4a918e2c054965e1871bde24216423dbf7df9c Mon Sep 17 00:00:00 2001 From: Paul Harris Date: Thu, 5 Dec 2024 07:50:15 +1000 Subject: [PATCH 11/20] fix event types message (#8885) Signed-off-by: Paul Harris --- .../teku/beaconrestapi/beacon/paths/_eth_v1_events.json | 2 +- .../tech/pegasys/teku/beaconrestapi/BeaconRestApiTypes.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/data/beaconrestapi/src/integration-test/resources/tech/pegasys/teku/beaconrestapi/beacon/paths/_eth_v1_events.json b/data/beaconrestapi/src/integration-test/resources/tech/pegasys/teku/beaconrestapi/beacon/paths/_eth_v1_events.json index c3806e52186..aaede887f56 100644 --- a/data/beaconrestapi/src/integration-test/resources/tech/pegasys/teku/beaconrestapi/beacon/paths/_eth_v1_events.json +++ b/data/beaconrestapi/src/integration-test/resources/tech/pegasys/teku/beaconrestapi/beacon/paths/_eth_v1_events.json @@ -9,7 +9,7 @@ "in" : "query", "schema" : { "type" : "string", - "description" : "Event types to subscribe to. Supported event types: [`attestation`, `attester_slashing`, `blob_sidecar`, `block_gossip`, `block`, `bls_to_execution_change`, `chain_reorg`, `contribution_and_proof`, `finalized_checkpoint`, `head`, `payload_attributes`, `proposer_slashing`, `single_attestation`, `sync_state`, `voluntary_exit`", + "description" : "Event types to subscribe to. Supported event types: [`attestation`, `attester_slashing`, `blob_sidecar`, `block_gossip`, `block`, `bls_to_execution_change`, `chain_reorg`, `contribution_and_proof`, `finalized_checkpoint`, `head`, `payload_attributes`, `proposer_slashing`, `single_attestation`, `sync_state`, `voluntary_exit`]", "example" : "head" } } ], diff --git a/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/BeaconRestApiTypes.java b/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/BeaconRestApiTypes.java index d136dc62da7..7267cf33814 100644 --- a/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/BeaconRestApiTypes.java +++ b/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/BeaconRestApiTypes.java @@ -215,7 +215,8 @@ public class BeaconRestApiTypes { + EnumSet.allOf(EventType.class).stream() .map(val -> "`" + val.toString() + "`") .sorted() - .collect(Collectors.joining(", ")), + .collect(Collectors.joining(", ")) + + "]", "head")); public static final SerializableTypeDefinition ROOT_TYPE = From fbf7359875db2bf8aed32f2de8c0401522914d56 Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Thu, 5 Dec 2024 23:19:29 +0100 Subject: [PATCH 12/20] Implements SingleAttestation handling (#8884) --- .../chains/ThrottlingSyncSource.java | 8 + .../v1/events/EventSubscriptionManager.java | 8 +- .../attestation/ValidatableAttestation.java | 30 ++- .../logic/common/util/AttestationUtil.java | 27 ++- .../electra/util/AttestationUtilElectra.java | 163 +++++++++++++++ .../common/util/AttestationUtilTest.java | 191 ++++++++++++++++-- .../teku/spec/util/DataStructureUtil.java | 15 +- .../validation/AttestationValidator.java | 25 ++- .../AggregatingAttestationPoolTest.java | 38 +++- .../MatchingDataAttestationGroupTest.java | 36 +++- .../AttestationSubnetSubscriptions.java | 15 +- .../AttestationSubnetSubscriptionsTest.java | 30 ++- .../AttestationProductionDuty.java | 71 ++++++- .../duties/AttestationProductionDutyTest.java | 134 +++++++++--- 14 files changed, 706 insertions(+), 85 deletions(-) diff --git a/beacon/sync/src/main/java/tech/pegasys/teku/beacon/sync/forward/multipeer/chains/ThrottlingSyncSource.java b/beacon/sync/src/main/java/tech/pegasys/teku/beacon/sync/forward/multipeer/chains/ThrottlingSyncSource.java index 2f6ab09a4e5..b8be60d4fb8 100644 --- a/beacon/sync/src/main/java/tech/pegasys/teku/beacon/sync/forward/multipeer/chains/ThrottlingSyncSource.java +++ b/beacon/sync/src/main/java/tech/pegasys/teku/beacon/sync/forward/multipeer/chains/ThrottlingSyncSource.java @@ -65,6 +65,10 @@ public SafeFuture requestBlocksByRange( LOG.debug("Sending request for {} blocks", count); return delegate.requestBlocksByRange(startSlot, count, listener); } else { + LOG.debug( + "Rate limiting request for {} blocks. Retry in {} seconds", + count, + PEER_REQUEST_DELAY.toSeconds()); return asyncRunner.runAfterDelay( () -> requestBlocksByRange(startSlot, count, listener), PEER_REQUEST_DELAY); } @@ -77,6 +81,10 @@ public SafeFuture requestBlobSidecarsByRange( LOG.debug("Sending request for {} blob sidecars", count); return delegate.requestBlobSidecarsByRange(startSlot, count, listener); } else { + LOG.debug( + "Rate limiting request for {} blob sidecars. Retry in {} seconds", + count, + PEER_REQUEST_DELAY.toSeconds()); return asyncRunner.runAfterDelay( () -> requestBlobSidecarsByRange(startSlot, count, listener), PEER_REQUEST_DELAY); } diff --git a/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java b/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java index fb0c4238425..640b1c87afc 100644 --- a/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java +++ b/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java @@ -43,6 +43,7 @@ import tech.pegasys.teku.spec.datastructures.attestation.ValidatableAttestation; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; +import tech.pegasys.teku.spec.datastructures.operations.Attestation; import tech.pegasys.teku.spec.datastructures.operations.AttesterSlashing; import tech.pegasys.teku.spec.datastructures.operations.ProposerSlashing; import tech.pegasys.teku.spec.datastructures.operations.SignedBlsToExecutionChange; @@ -211,12 +212,13 @@ protected void onSyncCommitteeContribution( } protected void onNewAttestation(final ValidatableAttestation attestation) { - if (!attestation.getAttestation().isSingleAttestation()) { - final AttestationEvent attestationEvent = new AttestationEvent(attestation.getAttestation()); + final Attestation actualAttestation = attestation.getUnconvertedAttestation(); + if (!actualAttestation.isSingleAttestation()) { + final AttestationEvent attestationEvent = new AttestationEvent(actualAttestation); notifySubscribersOfEvent(EventType.attestation, attestationEvent); } else { final SingleAttestationEvent attestationEvent = - new SingleAttestationEvent(attestation.getAttestation().toSingleAttestationRequired()); + new SingleAttestationEvent(actualAttestation.toSingleAttestationRequired()); notifySubscribersOfEvent(EventType.single_attestation, attestationEvent); } } diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/attestation/ValidatableAttestation.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/attestation/ValidatableAttestation.java index 04ed1897c58..a276002d22e 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/attestation/ValidatableAttestation.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/attestation/ValidatableAttestation.java @@ -13,6 +13,8 @@ package tech.pegasys.teku.spec.datastructures.attestation; +import static com.google.common.base.Preconditions.checkState; + import com.google.common.annotations.VisibleForTesting; import com.google.common.base.MoreObjects; import com.google.common.base.Objects; @@ -24,6 +26,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Supplier; import org.apache.tuweni.bytes.Bytes32; +import org.jetbrains.annotations.NotNull; import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.Spec; import tech.pegasys.teku.spec.constants.Domain; @@ -35,13 +38,17 @@ public class ValidatableAttestation { private final Spec spec; - private final Attestation attestation; private final Optional maybeAggregate; private final Supplier hashTreeRoot; private final AtomicBoolean gossiped = new AtomicBoolean(false); private final boolean producedLocally; private final OptionalInt receivedSubnetId; + private final Attestation unconvertedAttestation; + + @NotNull // will help us not forget to initialize if a new constructor is added + private volatile Attestation attestation; + private volatile boolean isValidIndexedAttestation = false; private volatile boolean acceptedAsGossip = false; @@ -49,6 +56,14 @@ public class ValidatableAttestation { private volatile Optional committeeShufflingSeed = Optional.empty(); private volatile Optional committeesSize = Optional.empty(); + public void convertToAggregatedFormatFromSingleAttestation( + final Attestation aggregatedFormatFromSingleAttestation) { + checkState( + attestation.isSingleAttestation(), + "Attestation must be a single attestation to convert to aggregated format"); + this.attestation = aggregatedFormatFromSingleAttestation; + } + public static ValidatableAttestation from(final Spec spec, final Attestation attestation) { return new ValidatableAttestation( spec, attestation, Optional.empty(), OptionalInt.empty(), false); @@ -113,6 +128,7 @@ private ValidatableAttestation( this.spec = spec; this.maybeAggregate = aggregateAndProof; this.attestation = attestation; + this.unconvertedAttestation = attestation; this.receivedSubnetId = receivedSubnetId; this.hashTreeRoot = Suppliers.memoize(attestation::hashTreeRoot); this.producedLocally = producedLocally; @@ -128,6 +144,7 @@ private ValidatableAttestation( this.spec = spec; this.maybeAggregate = aggregateAndProof; this.attestation = attestation; + this.unconvertedAttestation = attestation; this.receivedSubnetId = receivedSubnetId; this.hashTreeRoot = Suppliers.memoize(attestation::hashTreeRoot); this.producedLocally = producedLocally; @@ -178,7 +195,7 @@ public void saveCommitteeShufflingSeedAndCommitteesSize(final BeaconState state) saveCommitteeShufflingSeed(state); // The committees size is only required when the committee_bits field is present in the // Attestation - if (attestation.requiresCommitteeBits()) { + if (attestation.isSingleAttestation() || attestation.requiresCommitteeBits()) { saveCommitteesSize(state); } } @@ -216,10 +233,15 @@ public boolean isAggregate() { return maybeAggregate.isPresent(); } + @NotNull public Attestation getAttestation() { return attestation; } + public Attestation getUnconvertedAttestation() { + return unconvertedAttestation; + } + public SignedAggregateAndProof getSignedAggregateAndProof() { return maybeAggregate.orElseThrow( () -> new UnsupportedOperationException("ValidatableAttestation is not an aggregate.")); @@ -246,10 +268,9 @@ public boolean equals(final Object o) { if (this == o) { return true; } - if (!(o instanceof ValidatableAttestation)) { + if (!(o instanceof ValidatableAttestation that)) { return false; } - ValidatableAttestation that = (ValidatableAttestation) o; return Objects.equal(getAttestation(), that.getAttestation()) && Objects.equal(maybeAggregate, that.maybeAggregate); } @@ -273,6 +294,7 @@ public String toString() { .add("committeeShufflingSeed", committeeShufflingSeed) .add("committeesSize", committeesSize) .add("receivedSubnetId", receivedSubnetId) + .add("unconvertedAttestation", unconvertedAttestation) .toString(); } } diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/common/util/AttestationUtil.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/common/util/AttestationUtil.java index 5da5b45f023..ecd8906aa7b 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/common/util/AttestationUtil.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/common/util/AttestationUtil.java @@ -252,24 +252,39 @@ public SafeFuture isValidIndexedAttestationAsync( AttestationProcessingResult.invalid("Attesting indices include non-existent validator")); } - final BLSSignature signature = indexedAttestation.getSignature(); + return validateAttestationDataSignature( + fork, + state, + pubkeys, + indexedAttestation.getSignature(), + indexedAttestation.getData(), + signatureVerifier); + } + + protected SafeFuture validateAttestationDataSignature( + final Fork fork, + final BeaconState state, + final List publicKeys, + final BLSSignature signature, + final AttestationData attestationData, + final AsyncBLSSignatureVerifier signatureVerifier) { + final Bytes32 domain = beaconStateAccessors.getDomain( Domain.BEACON_ATTESTER, - indexedAttestation.getData().getTarget().getEpoch(), + attestationData.getTarget().getEpoch(), fork, state.getGenesisValidatorsRoot()); - final Bytes signingRoot = miscHelpers.computeSigningRoot(indexedAttestation.getData(), domain); + final Bytes signingRoot = miscHelpers.computeSigningRoot(attestationData, domain); return signatureVerifier - .verify(pubkeys, signingRoot, signature) + .verify(publicKeys, signingRoot, signature) .thenApply( isValidSignature -> { if (isValidSignature) { return AttestationProcessingResult.SUCCESSFUL; } else { - LOG.debug( - "AttestationUtil.is_valid_indexed_attestation: Verify aggregate signature"); + LOG.debug("AttestationUtil.validateAttestationDataSignature: Verify signature"); return AttestationProcessingResult.invalid("Signature is invalid"); } }); diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/electra/util/AttestationUtilElectra.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/electra/util/AttestationUtilElectra.java index 943536e7bf6..dfa67c18f36 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/electra/util/AttestationUtilElectra.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/electra/util/AttestationUtilElectra.java @@ -13,23 +13,41 @@ package tech.pegasys.teku.spec.logic.versions.electra.util; +import static com.google.common.base.Preconditions.checkArgument; +import static tech.pegasys.teku.infrastructure.async.SafeFuture.completedFuture; + import it.unimi.dsi.fastutil.ints.IntArrayList; import it.unimi.dsi.fastutil.ints.IntList; import java.util.List; +import java.util.Optional; import java.util.stream.IntStream; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import tech.pegasys.teku.bls.BLSPublicKey; +import tech.pegasys.teku.infrastructure.async.SafeFuture; import tech.pegasys.teku.infrastructure.ssz.collections.SszBitlist; import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.config.SpecConfig; +import tech.pegasys.teku.spec.datastructures.attestation.ValidatableAttestation; import tech.pegasys.teku.spec.datastructures.blocks.BeaconBlockSummary; import tech.pegasys.teku.spec.datastructures.operations.Attestation; import tech.pegasys.teku.spec.datastructures.operations.AttestationData; +import tech.pegasys.teku.spec.datastructures.operations.IndexedAttestation; +import tech.pegasys.teku.spec.datastructures.operations.IndexedAttestationSchema; +import tech.pegasys.teku.spec.datastructures.operations.SingleAttestation; +import tech.pegasys.teku.spec.datastructures.operations.versions.electra.AttestationElectraSchema; +import tech.pegasys.teku.spec.datastructures.state.Fork; import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconState; +import tech.pegasys.teku.spec.datastructures.util.AttestationProcessingResult; import tech.pegasys.teku.spec.logic.common.helpers.BeaconStateAccessors; import tech.pegasys.teku.spec.logic.common.helpers.MiscHelpers; +import tech.pegasys.teku.spec.logic.common.util.AsyncBLSSignatureVerifier; import tech.pegasys.teku.spec.logic.versions.deneb.util.AttestationUtilDeneb; import tech.pegasys.teku.spec.schemas.SchemaDefinitions; public class AttestationUtilElectra extends AttestationUtilDeneb { + private static final Logger LOG = LogManager.getLogger(); + public AttestationUtilElectra( final SpecConfig specConfig, final SchemaDefinitions schemaDefinitions, @@ -93,4 +111,149 @@ public AttestationData getGenericAttestationData( final UInt64 committeeIndex) { return super.getGenericAttestationData(slot, state, block, UInt64.ZERO); } + + @Override + public IndexedAttestation getIndexedAttestation( + final BeaconState state, final Attestation attestation) { + if (attestation.isSingleAttestation()) { + return getIndexedAttestationFromSingleAttestation(attestation.toSingleAttestationRequired()); + } + return super.getIndexedAttestation(state, attestation); + } + + private IndexedAttestation getIndexedAttestationFromSingleAttestation( + final SingleAttestation attestation) { + final IndexedAttestationSchema indexedAttestationSchema = + schemaDefinitions.getIndexedAttestationSchema(); + + return indexedAttestationSchema.create( + indexedAttestationSchema + .getAttestingIndicesSchema() + .of(attestation.getValidatorIndexRequired()), + attestation.getData(), + attestation.getSignature()); + } + + @Override + public SafeFuture isValidIndexedAttestationAsync( + final Fork fork, + final BeaconState state, + final ValidatableAttestation attestation, + final AsyncBLSSignatureVerifier blsSignatureVerifier) { + + if (!attestation.getAttestation().isSingleAttestation()) { + // we can use the default implementation for aggregate attestation + return super.isValidIndexedAttestationAsync(fork, state, attestation, blsSignatureVerifier); + } + + // single attestation flow + + // 1. verify signature first + // 2. verify call getSingleAttestationAggregationBits which also validates the validatorIndex + // and the committee against the state + // 3. convert attestation inside ValidatableAttestation to AttestationElectra + // 4. set the indexed attestation into ValidatableAttestation + // 5. set the attestation as valid indexed attestation + + return validateSingleAttestationSignature( + fork, + state, + attestation.getAttestation().toSingleAttestationRequired(), + blsSignatureVerifier) + .thenApply( + result -> { + if (result.isSuccessful()) { + final SingleAttestation singleAttestation = + attestation.getAttestation().toSingleAttestationRequired(); + final IndexedAttestation indexedAttestation = + getIndexedAttestationFromSingleAttestation(singleAttestation); + + final SszBitlist singleAttestationAggregationBits = + getSingleAttestationAggregationBits(state, singleAttestation); + + final Attestation convertedAttestation = + convertSingleAttestationToAggregated( + singleAttestation, singleAttestationAggregationBits); + + attestation.convertToAggregatedFormatFromSingleAttestation(convertedAttestation); + attestation.saveCommitteeShufflingSeedAndCommitteesSize(state); + attestation.setIndexedAttestation(indexedAttestation); + attestation.setValidIndexedAttestation(); + } + return result; + }) + .exceptionallyCompose( + err -> { + if (err.getCause() instanceof IllegalArgumentException) { + LOG.debug("on_attestation: Attestation is not valid: ", err); + return SafeFuture.completedFuture( + AttestationProcessingResult.invalid(err.getMessage())); + } else { + return SafeFuture.failedFuture(err); + } + }); + } + + private Attestation convertSingleAttestationToAggregated( + final SingleAttestation singleAttestation, + final SszBitlist singleAttestationAggregationBits) { + final AttestationElectraSchema attestationElectraSchema = + schemaDefinitions.getAttestationSchema().toVersionElectra().orElseThrow(); + + return attestationElectraSchema.create( + singleAttestationAggregationBits, + singleAttestation.getData(), + singleAttestation.getAggregateSignature(), + attestationElectraSchema + .getCommitteeBitsSchema() + .orElseThrow() + .ofBits(singleAttestation.getFirstCommitteeIndex().intValue())); + } + + private SafeFuture validateSingleAttestationSignature( + final Fork fork, + final BeaconState state, + final SingleAttestation singleAttestation, + final AsyncBLSSignatureVerifier signatureVerifier) { + final Optional pubkey = + beaconStateAccessors.getValidatorPubKey( + state, singleAttestation.getValidatorIndexRequired()); + + if (pubkey.isEmpty()) { + return completedFuture( + AttestationProcessingResult.invalid("Attesting index include non-existent validator")); + } + + return validateAttestationDataSignature( + fork, + state, + List.of(pubkey.get()), + singleAttestation.getSignature(), + singleAttestation.getData(), + signatureVerifier); + } + + private SszBitlist getSingleAttestationAggregationBits( + final BeaconState state, final SingleAttestation singleAttestation) { + final IntList committee = + beaconStateAccessors.getBeaconCommittee( + state, + singleAttestation.getData().getSlot(), + singleAttestation.getFirstCommitteeIndex()); + + final int validatorIndex = singleAttestation.getValidatorIndexRequired().intValue(); + final int validatorCommitteeBit = committee.indexOf(validatorIndex); + + checkArgument( + validatorCommitteeBit >= 0, + "Validator index %s is not part of the committee %s", + validatorIndex, + singleAttestation.getFirstCommitteeIndex()); + + return schemaDefinitions + .toVersionElectra() + .orElseThrow() + .getAttestationSchema() + .createAggregationBitsOf(committee.size(), validatorCommitteeBit); + } } diff --git a/ethereum/spec/src/test/java/tech/pegasys/teku/spec/logic/common/util/AttestationUtilTest.java b/ethereum/spec/src/test/java/tech/pegasys/teku/spec/logic/common/util/AttestationUtilTest.java index 7d98b7fbc15..6ff63c43341 100644 --- a/ethereum/spec/src/test/java/tech/pegasys/teku/spec/logic/common/util/AttestationUtilTest.java +++ b/ethereum/spec/src/test/java/tech/pegasys/teku/spec/logic/common/util/AttestationUtilTest.java @@ -16,12 +16,21 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; +import static tech.pegasys.teku.spec.SpecMilestone.DENEB; +import static tech.pegasys.teku.spec.SpecMilestone.ELECTRA; +import static tech.pegasys.teku.spec.SpecMilestone.PHASE0; +import it.unimi.dsi.fastutil.ints.Int2IntOpenHashMap; import it.unimi.dsi.fastutil.ints.IntList; +import java.util.ArrayList; +import java.util.List; import java.util.Optional; import java.util.stream.IntStream; import org.apache.tuweni.bytes.Bytes; @@ -31,27 +40,31 @@ import tech.pegasys.teku.bls.BLSSignature; import tech.pegasys.teku.infrastructure.async.SafeFuture; import tech.pegasys.teku.infrastructure.ssz.Merkleizable; +import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.Spec; -import tech.pegasys.teku.spec.SpecMilestone; import tech.pegasys.teku.spec.SpecVersion; import tech.pegasys.teku.spec.TestSpecContext; import tech.pegasys.teku.spec.TestSpecInvocationContextProvider.SpecContext; import tech.pegasys.teku.spec.datastructures.attestation.ValidatableAttestation; import tech.pegasys.teku.spec.datastructures.operations.Attestation; import tech.pegasys.teku.spec.datastructures.operations.IndexedAttestation; +import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconState; import tech.pegasys.teku.spec.datastructures.util.AttestationProcessingResult; import tech.pegasys.teku.spec.logic.common.helpers.BeaconStateAccessors; import tech.pegasys.teku.spec.logic.common.helpers.MiscHelpers; +import tech.pegasys.teku.spec.logic.versions.deneb.util.AttestationUtilDeneb; +import tech.pegasys.teku.spec.logic.versions.electra.util.AttestationUtilElectra; import tech.pegasys.teku.spec.logic.versions.phase0.util.AttestationUtilPhase0; import tech.pegasys.teku.spec.util.DataStructureUtil; -@TestSpecContext(milestone = {SpecMilestone.PHASE0}) +@TestSpecContext(milestone = {PHASE0, DENEB, ELECTRA}) class AttestationUtilTest { private final MiscHelpers miscHelpers = mock(MiscHelpers.class); private final BeaconStateAccessors beaconStateAccessors = mock(BeaconStateAccessors.class); private final AsyncBLSSignatureVerifier asyncBLSSignatureVerifier = mock(AsyncBLSSignatureVerifier.class); + final Int2IntOpenHashMap committeesSize = new Int2IntOpenHashMap(); private Spec spec; private DataStructureUtil dataStructureUtil; @@ -60,11 +73,13 @@ class AttestationUtilTest { @BeforeEach void setUp(final SpecContext specContext) { - spec = specContext.getSpec(); + spec = spy(specContext.getSpec()); dataStructureUtil = specContext.getDataStructureUtil(); final SpecVersion specVersion = spec.forMilestone(specContext.getSpecMilestone()); - final IntList beaconCommittee = createBeaconCommittee(specVersion); - when(beaconStateAccessors.getBeaconCommittee(any(), any(), any())).thenReturn(beaconCommittee); + doAnswer(invocation -> createBeaconCommittee(specVersion, invocation.getArgument(2))) + .when(beaconStateAccessors) + .getBeaconCommittee(any(), any(), any()); + doAnswer(invocation -> committeesSize).when(spec).getBeaconCommitteesSize(any(), any()); when(beaconStateAccessors.getValidatorPubKey(any(), any())) .thenReturn(Optional.of(dataStructureUtil.randomPublicKey())); when(beaconStateAccessors.getDomain(any(), any(), any(), any())) @@ -84,15 +99,29 @@ void setUp(final SpecContext specContext) { beaconStateAccessors, miscHelpers); break; + case DENEB: + attestationUtil = + new AttestationUtilDeneb( + spec.getGenesisSpecConfig(), + specVersion.getSchemaDefinitions(), + beaconStateAccessors, + miscHelpers); + break; + case ELECTRA: + attestationUtil = + new AttestationUtilElectra( + spec.getGenesisSpecConfig(), + specVersion.getSchemaDefinitions(), + beaconStateAccessors, + miscHelpers); + break; default: throw new UnsupportedOperationException("unsupported milestone"); } } @TestTemplate - void noValidationIsDoneIfAttestationIsAlreadyValidAndIndexedAttestationIsPresent( - final SpecContext specContext) { - specContext.assumeIsOneOf(SpecMilestone.PHASE0); + void noValidationIsDoneIfAttestationIsAlreadyValidAndIndexedAttestationIsPresent() { final ValidatableAttestation validatableAttestation = ValidatableAttestation.from(spec, dataStructureUtil.randomAttestation()); validatableAttestation.setValidIndexedAttestation(); @@ -112,7 +141,6 @@ void noValidationIsDoneIfAttestationIsAlreadyValidAndIndexedAttestationIsPresent @TestTemplate void createsAndValidatesIndexedAttestation(final SpecContext specContext) { - specContext.assumeIsOneOf(SpecMilestone.PHASE0); final Attestation attestation = dataStructureUtil.randomAttestation(); final ValidatableAttestation validatableAttestation = ValidatableAttestation.from(spec, attestation); @@ -122,18 +150,90 @@ void createsAndValidatesIndexedAttestation(final SpecContext specContext) { assertThat(result).isCompletedWithValue(AttestationProcessingResult.SUCCESSFUL); + assertThat(validatableAttestation.getAttestation()) + .isSameAs(validatableAttestation.getUnconvertedAttestation()); + assertThat(validatableAttestation.getAttestation().isSingleAttestation()).isFalse(); assertThat(validatableAttestation.isValidIndexedAttestation()).isTrue(); assertThat(validatableAttestation.getIndexedAttestation()).isPresent(); assertThat(validatableAttestation.getCommitteeShufflingSeed()).isPresent(); - assertThat(validatableAttestation.getCommitteesSize()).isEmpty(); + if (specContext.getSpecMilestone().isGreaterThanOrEqualTo(ELECTRA)) { + assertThat(validatableAttestation.getCommitteesSize()).contains(committeesSize); + } else { + assertThat(validatableAttestation.getCommitteesSize()).isEmpty(); + } verify(asyncBLSSignatureVerifier).verify(anyList(), any(Bytes.class), any(BLSSignature.class)); } + @TestTemplate + void createsValidatesIndexedAttestationAndConvertsFromSingleAttestation( + final SpecContext specContext) { + specContext.assumeElectraActive(); + + final UInt64 committeeIndex = UInt64.valueOf(2); + final UInt64 validatorIndex = UInt64.valueOf(5); + + final Attestation attestation = + dataStructureUtil.randomSingleAttestation(validatorIndex, committeeIndex); + final ValidatableAttestation validatableAttestation = + ValidatableAttestation.fromNetwork(spec, attestation, 1); + + // we want to make sure that we do signature verification before we do the committee lookup, + // that may trigger shuffling calculation + // To do that, let's control signature verification result + final SafeFuture signatureVerificationResult = new SafeFuture<>(); + when(asyncBLSSignatureVerifier.verify(anyList(), any(Bytes.class), any(BLSSignature.class))) + .thenReturn(signatureVerificationResult); + + // let validator be the second in the committee + doAnswer(invocation -> IntList.of(validatorIndex.intValue() + 1, validatorIndex.intValue())) + .when(beaconStateAccessors) + .getBeaconCommittee(any(), any(), any()); + + final SafeFuture result = + executeValidation(validatableAttestation); + + // no beacon committee lookup before signature verification + verify(beaconStateAccessors, never()).getBeaconCommittee(any(), any(), any()); + + // validation still in progress + assertThat(result).isNotDone(); + + // signature verification completed + signatureVerificationResult.complete(true); + + // now we should have the beacon committee lookup + verify(beaconStateAccessors).getBeaconCommittee(any(), any(), any()); + + // now we have successful validation + assertThat(result).isCompletedWithValue(AttestationProcessingResult.SUCCESSFUL); + + assertThat(validatableAttestation.getUnconvertedAttestation().isSingleAttestation()) + .describedAs("Original is still single attestation") + .isTrue(); + assertThat(validatableAttestation.getAttestation().isSingleAttestation()) + .describedAs("Aggregated format is not single attestation") + .isFalse(); + assertThat(validatableAttestation.getAttestation().getAggregationBits().getBitCount()) + .describedAs("Refers to a single validator") + .isEqualTo(1); + assertThat(validatableAttestation.getAttestation().getCommitteeIndicesRequired()) + .describedAs("Refers to the correct committee") + .containsExactly(UInt64.valueOf(2)); + + assertThat(validatableAttestation.isValidIndexedAttestation()).isTrue(); + assertThat(validatableAttestation.getIndexedAttestation()).isPresent(); + assertThat(validatableAttestation.getCommitteeShufflingSeed()).isPresent(); + if (specContext.getSpecMilestone().isGreaterThanOrEqualTo(ELECTRA)) { + assertThat(validatableAttestation.getCommitteesSize()).contains(committeesSize); + } else { + assertThat(validatableAttestation.getCommitteesSize()).isEmpty(); + } + } + @TestTemplate void createsButDoesNotValidateIndexedAttestationBecauseItHasAlreadyBeenValidated( final SpecContext specContext) { - specContext.assumeIsOneOf(SpecMilestone.PHASE0); final Attestation attestation = dataStructureUtil.randomAttestation(); // reorged block does not require indexed attestation validation, however it requires the // creation of it @@ -148,11 +248,59 @@ void createsButDoesNotValidateIndexedAttestationBecauseItHasAlreadyBeenValidated assertThat(validatableAttestation.isValidIndexedAttestation()).isTrue(); assertThat(validatableAttestation.getIndexedAttestation()).isPresent(); assertThat(validatableAttestation.getCommitteeShufflingSeed()).isPresent(); - assertThat(validatableAttestation.getCommitteesSize()).isEmpty(); + if (specContext.getSpecMilestone().isGreaterThanOrEqualTo(ELECTRA)) { + assertThat(validatableAttestation.getCommitteesSize()).contains(committeesSize); + } else { + assertThat(validatableAttestation.getCommitteesSize()).isEmpty(); + } verifyNoInteractions(miscHelpers, asyncBLSSignatureVerifier); } + @TestTemplate + void getIndexedAttestationGetsBeaconCommitteeWhenAttestationIsNotSingle( + final SpecContext specContext) { + final Attestation attestation = dataStructureUtil.randomAttestation(); + final BeaconState beaconState = dataStructureUtil.randomBeaconState(); + final IndexedAttestation indexedAttestation = + attestationUtil.getIndexedAttestation(beaconState, attestation); + + if (specContext.getSpecMilestone().isGreaterThanOrEqualTo(ELECTRA)) { + attestation + .getCommitteeIndicesRequired() + .forEach( + index -> + verify(beaconStateAccessors) + .getBeaconCommittee(beaconState, attestation.getData().getSlot(), index)); + } else { + verify(beaconStateAccessors) + .getBeaconCommittee( + beaconState, attestation.getData().getSlot(), attestation.getData().getIndex()); + } + + assertThat(indexedAttestation.getData()).isEqualTo(attestation.getData()); + assertThat(indexedAttestation.getSignature()).isEqualTo(attestation.getAggregateSignature()); + } + + @TestTemplate + void getIndexedAttestationNoQueryToBeaconCommitteeWhenSingleAttestation( + final SpecContext specContext) { + specContext.assumeElectraActive(); + + final Attestation attestation = dataStructureUtil.randomSingleAttestation(); + final BeaconState beaconState = dataStructureUtil.randomBeaconState(); + + final IndexedAttestation indexedAttestation = + attestationUtil.getIndexedAttestation(beaconState, attestation); + + verify(beaconStateAccessors, never()).getBeaconCommittee(any(), any(), any()); + + assertThat(indexedAttestation.getAttestingIndices().streamUnboxed()) + .containsExactly(attestation.getValidatorIndexRequired()); + assertThat(indexedAttestation.getData()).isEqualTo(attestation.getData()); + assertThat(indexedAttestation.getSignature()).isEqualTo(attestation.getAggregateSignature()); + } + private SafeFuture executeValidation( final ValidatableAttestation validatableAttestation) { return attestationUtil.isValidIndexedAttestationAsync( @@ -162,9 +310,24 @@ private SafeFuture executeValidation( asyncBLSSignatureVerifier); } - private IntList createBeaconCommittee(final SpecVersion specVersion) { + static final List KNOWN_COMMITTEES = new ArrayList<>(); + + private IntList createBeaconCommittee( + final SpecVersion specVersion, final UInt64 committeeIndex) { + // we have to generate non overlapping committees, committeeIndex is a random UInt64, so we + // can't use it directly + int position = KNOWN_COMMITTEES.indexOf(committeeIndex); + if (position == -1) { + KNOWN_COMMITTEES.add(committeeIndex); + position = KNOWN_COMMITTEES.size() - 1; + } + + final int maxValidatorsPerCommittee = specVersion.getConfig().getMaxValidatorsPerCommittee(); + final int validatorIndexTranslation = position * maxValidatorsPerCommittee; + final int[] committee = - IntStream.rangeClosed(0, specVersion.getConfig().getMaxValidatorsPerCommittee() - 1) + IntStream.rangeClosed(0, maxValidatorsPerCommittee - 1) + .map(i -> i + validatorIndexTranslation) .toArray(); return IntList.of(committee); } diff --git a/ethereum/spec/src/testFixtures/java/tech/pegasys/teku/spec/util/DataStructureUtil.java b/ethereum/spec/src/testFixtures/java/tech/pegasys/teku/spec/util/DataStructureUtil.java index b1f39032a9a..122fbd9e617 100644 --- a/ethereum/spec/src/testFixtures/java/tech/pegasys/teku/spec/util/DataStructureUtil.java +++ b/ethereum/spec/src/testFixtures/java/tech/pegasys/teku/spec/util/DataStructureUtil.java @@ -825,7 +825,20 @@ public SingleAttestation randomSingleAttestation() { .toVersionElectra() .orElseThrow() .getSingleAttestationSchema() - .create(randomUInt64(), randomUInt64(), randomAttestationData(), randomSignature()); + .create( + randomUInt64(Integer.MAX_VALUE), + randomUInt64(Integer.MAX_VALUE), + randomAttestationData(), + randomSignature()); + } + + public SingleAttestation randomSingleAttestation( + final UInt64 validatorIndex, final UInt64 committeeIndex) { + return spec.getGenesisSchemaDefinitions() + .toVersionElectra() + .orElseThrow() + .getSingleAttestationSchema() + .create(committeeIndex, validatorIndex, randomAttestationData(), randomSignature()); } public Attestation randomAttestation(final long slot) { diff --git a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/validation/AttestationValidator.java b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/validation/AttestationValidator.java index f652f597585..041d1f9cb07 100644 --- a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/validation/AttestationValidator.java +++ b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/validation/AttestationValidator.java @@ -72,6 +72,11 @@ public SafeFuture validate( } private InternalValidationResult singleAttestationChecks(final Attestation attestation) { + // if it is a SingleAttestation type we are guaranteed to be a valid single attestation + if (attestation.isSingleAttestation()) { + return InternalValidationResult.ACCEPT; + } + // The attestation is unaggregated -- that is, it has exactly one participating validator // (len([bit for bit in attestation.aggregation_bits if bit == 0b1]) == 1). final int bitCount = attestation.getAggregationBits().getBitCount(); @@ -167,15 +172,17 @@ SafeFuture singleOrAggregateAttestationChecks attestation.getFirstCommitteeIndex(), receivedOnSubnetId.getAsInt())); } - // [REJECT] The number of aggregation bits matches the committee size - final IntList committee = - spec.getBeaconCommittee( - state, data.getSlot(), attestation.getFirstCommitteeIndex()); - if (committee.size() != attestation.getAggregationBits().size()) { - return completedFuture( - InternalValidationResultWithState.reject( - "Aggregation bit size %s is greater than committee size %s", - attestation.getAggregationBits().size(), committee.size())); + if (!attestation.isSingleAttestation()) { + // [REJECT] The number of aggregation bits matches the committee size + final IntList committee = + spec.getBeaconCommittee( + state, data.getSlot(), attestation.getFirstCommitteeIndex()); + if (committee.size() != attestation.getAggregationBits().size()) { + return completedFuture( + InternalValidationResultWithState.reject( + "Aggregation bit size %s is greater than committee size %s", + attestation.getAggregationBits().size(), committee.size())); + } } return spec.isValidIndexedAttestation( diff --git a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/attestation/AggregatingAttestationPoolTest.java b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/attestation/AggregatingAttestationPoolTest.java index 05d82c5c265..d5f7acae9ad 100644 --- a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/attestation/AggregatingAttestationPoolTest.java +++ b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/attestation/AggregatingAttestationPoolTest.java @@ -51,6 +51,8 @@ import tech.pegasys.teku.spec.datastructures.operations.Attestation; import tech.pegasys.teku.spec.datastructures.operations.AttestationData; import tech.pegasys.teku.spec.datastructures.operations.AttestationSchema; +import tech.pegasys.teku.spec.datastructures.operations.SingleAttestation; +import tech.pegasys.teku.spec.datastructures.operations.SingleAttestationSchema; import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconState; import tech.pegasys.teku.spec.logic.common.operations.validation.AttestationDataValidator.AttestationInvalidReason; import tech.pegasys.teku.spec.util.DataStructureUtil; @@ -630,9 +632,24 @@ private Attestation addAttestationFromValidators( final AttestationData data, final Spec spec, final int... validators) { - final Attestation attestation = createAttestation(data, spec, validators); - ValidatableAttestation validatableAttestation = - ValidatableAttestation.from(spec, attestation, committeeSizes); + final Attestation attestationFromValidators; + final Attestation attestation; + if (specMilestone.isGreaterThanOrEqualTo(ELECTRA) && validators.length == 1) { + attestationFromValidators = createSingleAttestation(data, validators[0]); + } else { + attestationFromValidators = createAttestation(data, spec, validators); + } + + final ValidatableAttestation validatableAttestation = + ValidatableAttestation.from(spec, attestationFromValidators, committeeSizes); + + if (attestationFromValidators.isSingleAttestation()) { + attestation = createAttestation(data, spec, validators); + validatableAttestation.convertToAggregatedFormatFromSingleAttestation(attestation); + } else { + attestation = attestationFromValidators; + } + validatableAttestation.saveCommitteeShufflingSeedAndCommitteesSize( dataStructureUtil.randomBeaconState(100, 15, data.getSlot())); aggregatingAttestationPool.add(validatableAttestation); @@ -643,6 +660,21 @@ private Attestation createAttestation(final AttestationData data, final int... v return createAttestation(data, spec, validators); } + private SingleAttestation createSingleAttestation( + final AttestationData data, final int validatorIndex) { + final SingleAttestationSchema attestationSchema = + spec.getGenesisSchemaDefinitions() + .toVersionElectra() + .orElseThrow() + .getSingleAttestationSchema(); + + return attestationSchema.create( + committeeIndex.orElseThrow(), + UInt64.valueOf(validatorIndex), + data, + dataStructureUtil.randomSignature()); + } + private Attestation createAttestation( final AttestationData data, final Spec spec, final int... validators) { final AttestationSchema attestationSchema = diff --git a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/attestation/MatchingDataAttestationGroupTest.java b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/attestation/MatchingDataAttestationGroupTest.java index c19dd5af076..a85cade50f7 100644 --- a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/attestation/MatchingDataAttestationGroupTest.java +++ b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/attestation/MatchingDataAttestationGroupTest.java @@ -353,7 +353,26 @@ private ValidatableAttestation createAttestation( final Optional committeeIndex, final int... validators) { final SszBitlist aggregationBits = attestationSchema.getAggregationBitsSchema().ofBits(10, validators); + final boolean isElectra = spec.atSlot(SLOT).getMilestone().isGreaterThanOrEqualTo(ELECTRA); final Supplier committeeBits; + final Optional singleAttestation; + final int resolvedCommitteeIndex = committeeIndex.orElse(0); + + if (validators.length == 1 && isElectra) { + singleAttestation = + Optional.of( + spec.getGenesisSchemaDefinitions() + .toVersionElectra() + .orElseThrow() + .getSingleAttestationSchema() + .create( + UInt64.valueOf(resolvedCommitteeIndex), + UInt64.valueOf(validators[0]), + attestationData, + dataStructureUtil.randomSignature())); + } else { + singleAttestation = Optional.empty(); + } if (spec.atSlot(SLOT).getMilestone().isGreaterThanOrEqualTo(ELECTRA)) { committeeBits = @@ -361,14 +380,21 @@ private ValidatableAttestation createAttestation( attestationSchema .getCommitteeBitsSchema() .orElseThrow() - .ofBits(committeeIndex.orElse(0)); + .ofBits(resolvedCommitteeIndex); } else { committeeBits = () -> null; } - return ValidatableAttestation.from( - spec, + + final Attestation attestation = attestationSchema.create( - aggregationBits, attestationData, dataStructureUtil.randomSignature(), committeeBits), - committeeSizes); + aggregationBits, attestationData, dataStructureUtil.randomSignature(), committeeBits); + + final ValidatableAttestation validatableAttestation = + ValidatableAttestation.from(spec, singleAttestation.orElse(attestation), committeeSizes); + + singleAttestation.ifPresent( + __ -> validatableAttestation.convertToAggregatedFormatFromSingleAttestation(attestation)); + + return validatableAttestation; } } diff --git a/networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/gossip/subnets/AttestationSubnetSubscriptions.java b/networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/gossip/subnets/AttestationSubnetSubscriptions.java index de1231d38c0..6df760cb70a 100644 --- a/networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/gossip/subnets/AttestationSubnetSubscriptions.java +++ b/networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/gossip/subnets/AttestationSubnetSubscriptions.java @@ -30,6 +30,8 @@ import tech.pegasys.teku.spec.datastructures.operations.Attestation; import tech.pegasys.teku.spec.datastructures.operations.AttestationSchema; import tech.pegasys.teku.spec.datastructures.state.ForkInfo; +import tech.pegasys.teku.spec.schemas.SchemaDefinitions; +import tech.pegasys.teku.spec.schemas.SchemaDefinitionsElectra; import tech.pegasys.teku.statetransition.util.DebugDataDumper; import tech.pegasys.teku.storage.client.RecentChainData; @@ -58,8 +60,14 @@ public AttestationSubnetSubscriptions( this.recentChainData = recentChainData; this.processor = processor; this.forkInfo = forkInfo; + final SchemaDefinitions schemaDefinitions = + spec.atEpoch(forkInfo.getFork().getEpoch()).getSchemaDefinitions(); attestationSchema = - spec.atEpoch(forkInfo.getFork().getEpoch()).getSchemaDefinitions().getAttestationSchema(); + schemaDefinitions + .toVersionElectra() + .>map( + SchemaDefinitionsElectra::getSingleAttestationSchema) + .orElse(schemaDefinitions.getAttestationSchema()); this.debugDataDumper = debugDataDumper; } @@ -110,4 +118,9 @@ private SafeFuture> computeSubnetForAttestation(final Attestat state.map( s -> recentChainData.getSpec().computeSubnetForAttestation(s, attestation))); } + + @VisibleForTesting + AttestationSchema getAttestationSchema() { + return attestationSchema; + } } diff --git a/networking/eth2/src/test/java/tech/pegasys/teku/networking/eth2/gossip/subnets/AttestationSubnetSubscriptionsTest.java b/networking/eth2/src/test/java/tech/pegasys/teku/networking/eth2/gossip/subnets/AttestationSubnetSubscriptionsTest.java index 4f9d5256248..fae952c80e1 100644 --- a/networking/eth2/src/test/java/tech/pegasys/teku/networking/eth2/gossip/subnets/AttestationSubnetSubscriptionsTest.java +++ b/networking/eth2/src/test/java/tech/pegasys/teku/networking/eth2/gossip/subnets/AttestationSubnetSubscriptionsTest.java @@ -14,6 +14,7 @@ package tech.pegasys.teku.networking.eth2.gossip.subnets; import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.ArgumentMatchers.contains; @@ -37,17 +38,17 @@ import tech.pegasys.teku.spec.datastructures.attestation.ValidatableAttestation; import tech.pegasys.teku.spec.datastructures.operations.Attestation; import tech.pegasys.teku.spec.util.DataStructureUtil; -import tech.pegasys.teku.statetransition.BeaconChainUtil; import tech.pegasys.teku.statetransition.util.DebugDataDumper; -import tech.pegasys.teku.storage.client.MemoryOnlyRecentChainData; import tech.pegasys.teku.storage.client.RecentChainData; +import tech.pegasys.teku.storage.storageSystem.InMemoryStorageSystemBuilder; +import tech.pegasys.teku.storage.storageSystem.StorageSystem; public class AttestationSubnetSubscriptionsTest { private final Spec spec = TestSpecFactory.createMinimalPhase0(); - + private final StorageSystem storageSystem = InMemoryStorageSystemBuilder.buildDefault(spec); private final DataStructureUtil dataStructureUtil = new DataStructureUtil(spec); private final StubAsyncRunner asyncRunner = new StubAsyncRunner(); - private final RecentChainData recentChainData = MemoryOnlyRecentChainData.create(spec); + private final RecentChainData recentChainData = storageSystem.recentChainData(); private final GossipNetwork gossipNetwork = mock(GossipNetwork.class); private final GossipEncoding gossipEncoding = GossipEncoding.SSZ_SNAPPY; @@ -59,7 +60,7 @@ public class AttestationSubnetSubscriptionsTest { @BeforeEach void setUp() { - BeaconChainUtil.create(spec, 0, recentChainData).initializeStorage(); + storageSystem.chainUpdater().initializeGenesis(); subnetSubscriptions = new AttestationSubnetSubscriptions( spec, @@ -148,6 +149,25 @@ void shouldUnsubscribeFromOnlyCommitteeOnSubnet() { verify(topicChannel).close(); } + @Test + void shouldCreateHandlerForSingleAttestationWhenMilestoneIsElectra() { + final Spec spec = TestSpecFactory.createMinimalElectra(); + final StorageSystem storageSystem = InMemoryStorageSystemBuilder.buildDefault(spec); + storageSystem.chainUpdater().initializeGenesis(); + subnetSubscriptions = + new AttestationSubnetSubscriptions( + spec, + asyncRunner, + gossipNetwork, + gossipEncoding, + storageSystem.recentChainData(), + processor, + storageSystem.recentChainData().getCurrentForkInfo().orElseThrow(), + DebugDataDumper.NOOP); + assertDoesNotThrow( + () -> subnetSubscriptions.getAttestationSchema().toSingleAttestationSchemaRequired()); + } + private int computeSubnetId(final Attestation attestation) { return spec.computeSubnetForAttestation( safeJoin(recentChainData.getBestState().orElseThrow()), attestation); diff --git a/validator/client/src/main/java/tech/pegasys/teku/validator/client/duties/attestations/AttestationProductionDuty.java b/validator/client/src/main/java/tech/pegasys/teku/validator/client/duties/attestations/AttestationProductionDuty.java index c6d97eab41e..1f9661f660b 100644 --- a/validator/client/src/main/java/tech/pegasys/teku/validator/client/duties/attestations/AttestationProductionDuty.java +++ b/validator/client/src/main/java/tech/pegasys/teku/validator/client/duties/attestations/AttestationProductionDuty.java @@ -35,7 +35,9 @@ import tech.pegasys.teku.spec.datastructures.operations.Attestation; import tech.pegasys.teku.spec.datastructures.operations.AttestationData; import tech.pegasys.teku.spec.datastructures.operations.AttestationSchema; +import tech.pegasys.teku.spec.datastructures.operations.SingleAttestationSchema; import tech.pegasys.teku.spec.datastructures.state.ForkInfo; +import tech.pegasys.teku.spec.schemas.SchemaDefinitions; import tech.pegasys.teku.validator.api.ValidatorApiChannel; import tech.pegasys.teku.validator.client.ForkProvider; import tech.pegasys.teku.validator.client.Validator; @@ -136,19 +138,51 @@ private List>> produceAttestationsForCo CREATE_TOTAL); unsignedAttestationFuture.propagateTo(committee.getAttestationDataFuture()); + final SignedAttestationProducer signedAttestationProducer = + selectSignedAttestationProducer(slot); + return committee.getValidators().stream() .map( validator -> signAttestationForValidatorInCommittee( - slot, forkInfo, committeeIndex, validator, unsignedAttestationFuture)) + slot, + forkInfo, + committeeIndex, + validator, + signedAttestationProducer, + unsignedAttestationFuture)) .toList(); } + private SignedAttestationProducer selectSignedAttestationProducer(final UInt64 slot) { + final SchemaDefinitions schemaDefinitions = spec.atSlot(slot).getSchemaDefinitions(); + + return schemaDefinitions + .toVersionElectra() + .map( + schemaDefinitionsElectra -> + (attestationData, validator, signature) -> + createSignedSingleAttestation( + schemaDefinitionsElectra.getSingleAttestationSchema(), + attestationData, + validator, + signature)) + .orElseGet( + () -> + (attestationData, validator, signature) -> + createSignedAttestation( + schemaDefinitions.getAttestationSchema(), + attestationData, + validator, + signature)); + } + private SafeFuture> signAttestationForValidatorInCommittee( final UInt64 slot, final ForkInfo forkInfo, final int committeeIndex, final ValidatorWithAttestationDutyInfo validator, + final SignedAttestationProducer signedAttestationProducer, final SafeFuture> attestationDataFuture) { return attestationDataFuture .thenCompose( @@ -159,7 +193,11 @@ private SafeFuture> signAttestationForValidatorInC validateAttestationData(slot, attestationData); return validatorDutyMetrics.record( () -> - signAttestationForValidator(forkInfo, attestationData, validator), + signAttestationForValidator( + signedAttestationProducer, + forkInfo, + attestationData, + validator), this, ValidatorDutyMetricsSteps.SIGN); }) @@ -187,13 +225,17 @@ private static void validateAttestationData( } private SafeFuture> signAttestationForValidator( + final SignedAttestationProducer signedAttestationProducer, final ForkInfo forkInfo, final AttestationData attestationData, final ValidatorWithAttestationDutyInfo validator) { return validator .signer() .signAttestationData(attestationData, forkInfo) - .thenApply(signature -> createSignedAttestation(attestationData, validator, signature)) + .thenApply( + signature -> + signedAttestationProducer.createSignedAttestation( + attestationData, validator, signature)) .thenApply( attestation -> ProductionResult.success( @@ -201,11 +243,10 @@ private SafeFuture> signAttestationForValidator( } private Attestation createSignedAttestation( + final AttestationSchema attestationSchema, final AttestationData attestationData, final ValidatorWithAttestationDutyInfo validator, final BLSSignature signature) { - final AttestationSchema attestationSchema = - spec.atSlot(attestationData.getSlot()).getSchemaDefinitions().getAttestationSchema(); final SszBitlist aggregationBits = attestationSchema .getAggregationBitsSchema() @@ -220,4 +261,24 @@ private Attestation createSignedAttestation( return attestationSchema.create( aggregationBits, attestationData, signature, committeeBitsSupplier); } + + private Attestation createSignedSingleAttestation( + final SingleAttestationSchema attestationSchema, + final AttestationData attestationData, + final ValidatorWithAttestationDutyInfo validator, + final BLSSignature signature) { + return attestationSchema.create( + UInt64.valueOf(validator.committeeIndex()), + UInt64.valueOf(validator.validatorIndex()), + attestationData, + signature); + } + + @FunctionalInterface + private interface SignedAttestationProducer { + Attestation createSignedAttestation( + final AttestationData attestationData, + final ValidatorWithAttestationDutyInfo validator, + final BLSSignature signature); + } } diff --git a/validator/client/src/test/java/tech/pegasys/teku/validator/client/duties/AttestationProductionDutyTest.java b/validator/client/src/test/java/tech/pegasys/teku/validator/client/duties/AttestationProductionDutyTest.java index 6a2d0a9e215..2d3c3310df7 100644 --- a/validator/client/src/test/java/tech/pegasys/teku/validator/client/duties/AttestationProductionDutyTest.java +++ b/validator/client/src/test/java/tech/pegasys/teku/validator/client/duties/AttestationProductionDutyTest.java @@ -38,7 +38,6 @@ import java.util.List; import java.util.Optional; import java.util.Set; -import java.util.function.Supplier; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.TestTemplate; import org.mockito.ArgumentCaptor; @@ -47,7 +46,6 @@ import tech.pegasys.teku.infrastructure.logging.ValidatorLogger; import tech.pegasys.teku.infrastructure.metrics.StubMetricsSystem; import tech.pegasys.teku.infrastructure.ssz.collections.SszBitlist; -import tech.pegasys.teku.infrastructure.ssz.collections.SszBitvector; import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.Spec; import tech.pegasys.teku.spec.TestSpecContext; @@ -55,7 +53,9 @@ import tech.pegasys.teku.spec.datastructures.operations.Attestation; import tech.pegasys.teku.spec.datastructures.operations.AttestationData; import tech.pegasys.teku.spec.datastructures.operations.AttestationSchema; +import tech.pegasys.teku.spec.datastructures.operations.SingleAttestationSchema; import tech.pegasys.teku.spec.datastructures.state.ForkInfo; +import tech.pegasys.teku.spec.schemas.SchemaDefinitions; import tech.pegasys.teku.spec.signatures.Signer; import tech.pegasys.teku.spec.util.DataStructureUtil; import tech.pegasys.teku.validator.api.FileBackedGraffitiProvider; @@ -137,8 +137,10 @@ public void shouldFailWhenUnsignedAttestationCanNotBeCreated() { public void shouldPublishProducedAttestationsWhenSomeUnsignedAttestationsCanNotBeCreated() { final Validator validator1 = createValidator(); final Validator validator2 = createValidator(); + final int validator1Index = 10; final int validator1CommitteeIndex = 0; final int validator1CommitteePosition = 5; + final int validator2Index = 20; final int validator2CommitteeIndex = 1; final int validator2CommitteePosition = 3; final int validator2CommitteeSize = 8; @@ -148,6 +150,7 @@ public void shouldPublishProducedAttestationsWhenSomeUnsignedAttestationsCanNotB final Attestation expectedAttestation = expectSignAttestation( validator2, + validator2Index, validator2CommitteeIndex, validator2CommitteePosition, validator2CommitteeSize, @@ -155,13 +158,13 @@ public void shouldPublishProducedAttestationsWhenSomeUnsignedAttestationsCanNotB final SafeFuture> attestationResult1 = duty.addValidator( - validator1, validator1CommitteeIndex, validator1CommitteePosition, 10, 11); + validator1, validator1CommitteeIndex, validator1CommitteePosition, validator1Index, 11); final SafeFuture> attestationResult2 = duty.addValidator( validator2, validator2CommitteeIndex, validator2CommitteePosition, - 10, + validator2Index, validator2CommitteeSize); performAndReportDuty(); @@ -190,8 +193,10 @@ public void shouldPublishProducedAttestationsWhenSomeUnsignedAttestationsCanNotB public void shouldPublishProducedAttestationsWhenSomeUnsignedAttestationsFail() { final Validator validator1 = createValidator(); final Validator validator2 = createValidator(); + final int validator1Index = 10; final int validator1CommitteeIndex = 0; final int validator1CommitteePosition = 5; + final int validator2Index = 20; final int validator2CommitteeIndex = 1; final int validator2CommitteePosition = 3; final int validator2CommitteeSize = 12; @@ -202,6 +207,7 @@ public void shouldPublishProducedAttestationsWhenSomeUnsignedAttestationsFail() final Attestation expectedAttestation = expectSignAttestation( validator2, + validator2Index, validator2CommitteeIndex, validator2CommitteePosition, validator2CommitteeSize, @@ -209,13 +215,13 @@ public void shouldPublishProducedAttestationsWhenSomeUnsignedAttestationsFail() final SafeFuture> attestationResult1 = duty.addValidator( - validator1, validator1CommitteeIndex, validator1CommitteePosition, 10, 11); + validator1, validator1CommitteeIndex, validator1CommitteePosition, validator1Index, 11); final SafeFuture> attestationResult2 = duty.addValidator( validator2, validator2CommitteeIndex, validator2CommitteePosition, - 10, + validator2Index, validator2CommitteeSize); performAndReportDuty(); @@ -243,8 +249,10 @@ public void shouldPublishProducedAttestationsWhenSignerFailsForSomeAttestations( final Validator validator1 = createValidator(); final Validator validator2 = createValidator(); final int committeeIndex = 0; + final int validator1Index = 10; final int validator1CommitteePosition = 5; final int validator2CommitteePosition = 3; + final int validator2Index = 20; final int validator1CommitteeSize = 23; final int validator2CommitteeSize = 39; final AttestationData attestationData = expectCreateAttestationData(committeeIndex); @@ -254,6 +262,7 @@ public void shouldPublishProducedAttestationsWhenSignerFailsForSomeAttestations( final Attestation expectedAttestation = expectSignAttestation( validator2, + validator2Index, committeeIndex, validator2CommitteePosition, validator2CommitteeSize, @@ -261,10 +270,18 @@ public void shouldPublishProducedAttestationsWhenSignerFailsForSomeAttestations( final SafeFuture> attestationResult1 = duty.addValidator( - validator1, committeeIndex, validator1CommitteePosition, 10, validator1CommitteeSize); + validator1, + committeeIndex, + validator1CommitteePosition, + validator1Index, + validator1CommitteeSize); final SafeFuture> attestationResult2 = duty.addValidator( - validator2, committeeIndex, validator2CommitteePosition, 10, validator2CommitteeSize); + validator2, + committeeIndex, + validator2CommitteePosition, + validator2Index, + validator2CommitteeSize); performAndReportDuty(); assertThat(attestationResult1).isCompletedWithValue(Optional.of(attestationData)); @@ -288,6 +305,7 @@ public void shouldPublishProducedAttestationsWhenSignerFailsForSomeAttestations( @TestTemplate public void shouldCreateAttestationForSingleValidator() { + final int validatorIndex = 10; final int committeeIndex = 3; final int committeePosition = 6; final int committeeSize = 22; @@ -296,10 +314,16 @@ public void shouldCreateAttestationForSingleValidator() { final AttestationData attestationData = expectCreateAttestationData(committeeIndex); final Attestation expectedAttestation = expectSignAttestation( - validator, committeeIndex, committeePosition, committeeSize, attestationData); + validator, + validatorIndex, + committeeIndex, + committeePosition, + committeeSize, + attestationData); final SafeFuture> attestationResult = - duty.addValidator(validator, committeeIndex, committeePosition, 10, committeeSize); + duty.addValidator( + validator, committeeIndex, committeePosition, validatorIndex, committeeSize); performAndReportDuty(); assertThat(attestationResult).isCompletedWithValue(Optional.of(attestationData)); @@ -316,6 +340,7 @@ public void shouldCreateAttestationForSingleValidator() { @TestTemplate void shouldReportFailureWhenAttestationIsInvalid() { + final int validatorIndex = 10; final int committeeIndex = 3; final int committeePosition = 6; final int committeeSize = 22; @@ -324,7 +349,7 @@ void shouldReportFailureWhenAttestationIsInvalid() { final AttestationData attestationData = expectCreateAttestationData(committeeIndex); final Attestation expectedAttestation = expectSignAttestation( - validator, committeeIndex, committeePosition, committeeSize, attestationData); + validator, 10, committeeIndex, committeePosition, committeeSize, attestationData); when(validatorApiChannel.sendSignedAttestations(List.of(expectedAttestation))) .thenReturn( @@ -332,7 +357,8 @@ void shouldReportFailureWhenAttestationIsInvalid() { List.of(new SubmitDataError(UInt64.ZERO, "Naughty attestation")))); final SafeFuture> attestationResult = - duty.addValidator(validator, committeeIndex, committeePosition, 10, committeeSize); + duty.addValidator( + validator, committeeIndex, committeePosition, validatorIndex, committeeSize); performAndReportDuty(); assertThat(attestationResult).isCompletedWithValue(Optional.of(attestationData)); @@ -358,6 +384,9 @@ void shouldReportFailureWhenAttestationIsInvalid() { public void shouldCreateAttestationForMultipleValidatorsInSameCommittee() { final int committeeIndex = 3; final int committeeSize = 33; + final int validator1Index = 10; + final int validator2Index = 20; + final int validator3Index = 30; final int validator1CommitteePosition = 6; final int validator2CommitteePosition = 2; final int validator3CommitteePosition = 5; @@ -369,6 +398,7 @@ public void shouldCreateAttestationForMultipleValidatorsInSameCommittee() { final Attestation expectedAttestation1 = expectSignAttestation( validator1, + validator1Index, committeeIndex, validator1CommitteePosition, committeeSize, @@ -376,6 +406,7 @@ public void shouldCreateAttestationForMultipleValidatorsInSameCommittee() { final Attestation expectedAttestation2 = expectSignAttestation( validator2, + validator2Index, committeeIndex, validator2CommitteePosition, committeeSize, @@ -383,6 +414,7 @@ public void shouldCreateAttestationForMultipleValidatorsInSameCommittee() { final Attestation expectedAttestation3 = expectSignAttestation( validator3, + validator3Index, committeeIndex, validator3CommitteePosition, committeeSize, @@ -390,13 +422,25 @@ public void shouldCreateAttestationForMultipleValidatorsInSameCommittee() { final SafeFuture> attestationResult1 = duty.addValidator( - validator1, committeeIndex, validator1CommitteePosition, 10, committeeSize); + validator1, + committeeIndex, + validator1CommitteePosition, + validator1Index, + committeeSize); final SafeFuture> attestationResult2 = duty.addValidator( - validator2, committeeIndex, validator2CommitteePosition, 10, committeeSize); + validator2, + committeeIndex, + validator2CommitteePosition, + validator2Index, + committeeSize); final SafeFuture> attestationResult3 = duty.addValidator( - validator3, committeeIndex, validator3CommitteePosition, 10, committeeSize); + validator3, + committeeIndex, + validator3CommitteePosition, + validator3Index, + committeeSize); performAndReportDuty(); assertThat(attestationResult1).isCompletedWithValue(Optional.of(attestationData)); assertThat(attestationResult2).isCompletedWithValue(Optional.of(attestationData)); @@ -428,6 +472,9 @@ public void shouldCreateAttestationForMultipleValidatorsInDifferentCommittees() final int committeeIndex2 = 1; final int committeeSize1 = 15; final int committeeSize2 = 20; + final int validator1Index = 10; + final int validator2Index = 20; + final int validator3Index = 30; final int validator1CommitteePosition = 6; final int validator2CommitteePosition = 2; final int validator3CommitteePosition = 5; @@ -440,6 +487,7 @@ public void shouldCreateAttestationForMultipleValidatorsInDifferentCommittees() final Attestation expectedAttestation1 = expectSignAttestation( validator1, + validator1Index, committeeIndex1, validator1CommitteePosition, committeeSize1, @@ -447,6 +495,7 @@ public void shouldCreateAttestationForMultipleValidatorsInDifferentCommittees() final Attestation expectedAttestation2 = expectSignAttestation( validator2, + validator2Index, committeeIndex2, validator2CommitteePosition, committeeSize2, @@ -454,6 +503,7 @@ public void shouldCreateAttestationForMultipleValidatorsInDifferentCommittees() final Attestation expectedAttestation3 = expectSignAttestation( validator3, + validator3Index, committeeIndex1, validator3CommitteePosition, committeeSize1, @@ -461,13 +511,25 @@ public void shouldCreateAttestationForMultipleValidatorsInDifferentCommittees() final SafeFuture> attestationResult1 = duty.addValidator( - validator1, committeeIndex1, validator1CommitteePosition, 10, committeeSize1); + validator1, + committeeIndex1, + validator1CommitteePosition, + validator1Index, + committeeSize1); final SafeFuture> attestationResult2 = duty.addValidator( - validator2, committeeIndex2, validator2CommitteePosition, 10, committeeSize2); + validator2, + committeeIndex2, + validator2CommitteePosition, + validator2Index, + committeeSize2); final SafeFuture> attestationResult3 = duty.addValidator( - validator3, committeeIndex1, validator3CommitteePosition, 10, committeeSize1); + validator3, + committeeIndex1, + validator3CommitteePosition, + validator3Index, + committeeSize1); performAndReportDuty(); assertThat(attestationResult1).isCompletedWithValue(Optional.of(unsignedAttestation1)); @@ -507,6 +569,7 @@ private Validator createValidator() { private Attestation expectSignAttestation( final Validator validator, + final int validatorIndex, final int committeeIndex, final int committeePosition, final int committeeSize, @@ -516,7 +579,12 @@ private Attestation expectSignAttestation( .thenReturn(completedFuture(signature)); return createExpectedAttestation( - attestationData, committeeIndex, committeePosition, committeeSize, signature); + attestationData, + validatorIndex, + committeeIndex, + committeePosition, + committeeSize, + signature); } private AttestationData expectCreateAttestationData(final int committeeIndex) { @@ -528,26 +596,34 @@ private AttestationData expectCreateAttestationData(final int committeeIndex) { private Attestation createExpectedAttestation( final AttestationData attestationData, + final int validatorIndex, final int committeeIndex, final int committeePosition, final int committeeSize, final BLSSignature signature) { + final SchemaDefinitions schemaDefinitions = + spec.atSlot(attestationData.getSlot()).getSchemaDefinitions(); + + if (schemaDefinitions.toVersionElectra().isPresent()) { + final SingleAttestationSchema attestationSchema = + schemaDefinitions.toVersionElectra().orElseThrow().getSingleAttestationSchema(); + + return attestationSchema.create( + UInt64.valueOf(committeeIndex), + UInt64.valueOf(validatorIndex), + attestationData, + signature); + } + + // Phase 0 + final AttestationSchema attestationSchema = spec.atSlot(attestationData.getSlot()).getSchemaDefinitions().getAttestationSchema(); final SszBitlist expectedAggregationBits = attestationSchema.getAggregationBitsSchema().ofBits(committeeSize, committeePosition); - final Supplier committeeBits; - - if (spec.atSlot(attestationData.getSlot()).getMilestone().isGreaterThanOrEqualTo(ELECTRA)) { - committeeBits = - () -> attestationSchema.getCommitteeBitsSchema().orElseThrow().ofBits(committeeIndex); - } else { - committeeBits = () -> null; - } - return attestationSchema.create( - expectedAggregationBits, attestationData, signature, committeeBits); + expectedAggregationBits, attestationData, signature, () -> null); } private void performAndReportDuty() { From a8c2e0e5238650ac16ac397ba2d21f8f99cb8e7a Mon Sep 17 00:00:00 2001 From: Lucas Saldanha Date: Fri, 6 Dec 2024 15:29:46 +1300 Subject: [PATCH 13/20] Electra ref-test alpha.9 update (#8882) Co-authored by @StefanBratanov Co-authored by @gfukushima --- build.gradle | 2 +- .../operations/OperationsTestExecutor.java | 9 ++++++++ .../execution/ExpectedWithdrawals.java | 17 +++++++++++---- .../electra/block/BlockProcessorElectra.java | 14 +++++++++++++ .../forktransition/ElectraStateUpgrade.java | 21 ++++++++++--------- .../ElectraStateUpgradeTest.java | 5 ++++- .../tech/pegasys/teku/fuzz/FuzzUtilTest.java | 3 +++ 7 files changed, 55 insertions(+), 16 deletions(-) diff --git a/build.gradle b/build.gradle index d6df0daff8e..b7b39b6d4c0 100644 --- a/build.gradle +++ b/build.gradle @@ -325,7 +325,7 @@ allprojects { } def nightly = System.getenv("NIGHTLY") != null -def refTestVersion = nightly ? "nightly" : "v1.5.0-alpha.8" +def refTestVersion = nightly ? "nightly" : "v1.5.0-alpha.9" def blsRefTestVersion = 'v0.1.2' def slashingProtectionInterchangeRefTestVersion = 'v5.3.0' def refTestBaseUrl = 'https://github.com/ethereum/consensus-spec-tests/releases/download' diff --git a/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/common/operations/OperationsTestExecutor.java b/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/common/operations/OperationsTestExecutor.java index df11615cc16..3651eae033d 100644 --- a/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/common/operations/OperationsTestExecutor.java +++ b/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/common/operations/OperationsTestExecutor.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableMap; +import java.util.List; import java.util.Optional; import tech.pegasys.teku.ethtests.finder.TestDefinition; import tech.pegasys.teku.infrastructure.ssz.SszData; @@ -67,6 +68,9 @@ public class OperationsTestExecutor implements TestExecutor { public static final String EXPECTED_STATE_FILE = "post.ssz_snappy"; + // TODO remove https://github.com/Consensys/teku/issues/8892 + private static final List IGNORED_TEST = List.of("invalid_nonset_bits_for_one_committee"); + private enum Operation { ATTESTER_SLASHING, PROPOSER_SLASHING, @@ -144,6 +148,11 @@ public OperationsTestExecutor(final String dataFileName, final Operation operati @Override public void runTest(final TestDefinition testDefinition) throws Exception { + // TODO remove https://github.com/Consensys/teku/issues/8892 + if (IGNORED_TEST.contains(testDefinition.getTestName())) { + return; + } + final BeaconState preState = loadStateFromSsz(testDefinition, "pre.ssz_snappy"); final DefaultOperationProcessor standardProcessor = diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/execution/ExpectedWithdrawals.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/execution/ExpectedWithdrawals.java index d839ba747de..e8a701d4ddd 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/execution/ExpectedWithdrawals.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/execution/ExpectedWithdrawals.java @@ -164,7 +164,7 @@ private static WithdrawalSummary getPendingPartialWithdrawals( final List partialWithdrawals = new ArrayList<>(); final SszList pendingPartialWithdrawals = preState.getPendingPartialWithdrawals(); - int partialWithdrawalsCount = 0; + int processedPartialWithdrawalsCount = 0; UInt64 withdrawalIndex = preState.getNextWithdrawalIndex(); for (int i = 0; i < pendingPartialWithdrawals.size() && i < maxPendingPartialWithdrawals; i++) { @@ -198,9 +198,9 @@ private static WithdrawalSummary getPendingPartialWithdrawals( withdrawableBalance)); withdrawalIndex = withdrawalIndex.increment(); } - partialWithdrawalsCount++; + processedPartialWithdrawalsCount++; } - return new WithdrawalSummary(partialWithdrawals, partialWithdrawalsCount); + return new WithdrawalSummary(partialWithdrawals, processedPartialWithdrawalsCount); } // get_expected_withdrawals @@ -231,7 +231,16 @@ private static List getExpectedWithdrawals( for (int i = 0; i < bound; i++) { final Validator validator = validators.get(validatorIndex); if (predicates.hasExecutionWithdrawalCredential(validator)) { - final UInt64 balance = balances.get(validatorIndex).get(); + final UInt64 finalValidatorIndex = UInt64.valueOf(validatorIndex); + final UInt64 partiallyWithdrawalBalance = + partialWithdrawals.stream() + .filter( + partialWithdrawal -> + partialWithdrawal.getValidatorIndex().equals(finalValidatorIndex)) + .map(Withdrawal::getAmount) + .reduce(UInt64.ZERO, UInt64::plus); + final UInt64 balance = + balances.get(validatorIndex).get().minusMinZero(partiallyWithdrawalBalance); if (predicates.isFullyWithdrawableValidatorCredentialsChecked(validator, balance, epoch)) { expectedWithdrawals.add( diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/electra/block/BlockProcessorElectra.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/electra/block/BlockProcessorElectra.java index 49493302242..9bfe1da33b4 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/electra/block/BlockProcessorElectra.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/electra/block/BlockProcessorElectra.java @@ -520,6 +520,20 @@ private void processConsolidationRequest( return; } + // Verify the source has been active long enough + if (currentEpoch.isLessThan( + sourceValidator.getActivationEpoch().plus(specConfig.getShardCommitteePeriod()))) { + LOG.debug("process_consolidation_request: source has not been active long enough"); + return; + } + // Verify the source has no pending withdrawals in the queue + if (beaconStateAccessorsElectra + .getPendingBalanceToWithdraw(state, sourceValidatorIndex) + .isGreaterThan(ZERO)) { + LOG.debug("process_consolidation_request: source has pending withdrawals in the queue"); + return; + } + // Initiate source validator exit and append pending consolidation final UInt64 exitEpoch = beaconStateMutatorsElectra.computeConsolidationEpochAndUpdateChurn( diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/electra/forktransition/ElectraStateUpgrade.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/electra/forktransition/ElectraStateUpgrade.java index a5c71a7af76..f2e8251b733 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/electra/forktransition/ElectraStateUpgrade.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/electra/forktransition/ElectraStateUpgrade.java @@ -58,6 +58,7 @@ public BeaconStateElectra upgrade(final BeaconState preState) { final PredicatesElectra predicatesElectra = new PredicatesElectra(specConfig); final MiscHelpersElectra miscHelpersElectra = new MiscHelpersElectra(specConfig, predicatesElectra, schemaDefinitions); + final UInt64 activationExitEpoch = miscHelpersElectra.computeActivationExitEpoch(epoch); return BeaconStateElectra.required(schemaDefinitions.getBeaconStateSchema().createEmpty()) .updatedElectra( state -> { @@ -86,11 +87,10 @@ public BeaconStateElectra upgrade(final BeaconState preState) { state.setDepositBalanceToConsume(UInt64.ZERO); state.setExitBalanceToConsume( beaconStateAccessors.getActivationExitChurnLimit(state)); - state.setEarliestExitEpoch(findEarliestExitEpoch(state, epoch)); + state.setEarliestExitEpoch(findEarliestExitEpoch(state, activationExitEpoch)); state.setConsolidationBalanceToConsume( beaconStateAccessors.getConsolidationChurnLimit(state)); - state.setEarliestConsolidationEpoch( - miscHelpersElectra.computeActivationExitEpoch(epoch)); + state.setEarliestConsolidationEpoch(activationExitEpoch); final SszMutableList validators = state.getValidators(); @@ -120,12 +120,13 @@ public BeaconStateElectra upgrade(final BeaconState preState) { }); } - private UInt64 findEarliestExitEpoch(final BeaconState state, final UInt64 currentEpoch) { - return state.getValidators().stream() - .map(Validator::getExitEpoch) - .filter(exitEpoch -> !exitEpoch.equals(FAR_FUTURE_EPOCH)) - .max(UInt64::compareTo) - .orElse(currentEpoch) - .increment(); + private UInt64 findEarliestExitEpoch(final BeaconState state, final UInt64 activationExitEpoch) { + final UInt64 maxExitEpochFromValidatorSet = + state.getValidators().stream() + .map(Validator::getExitEpoch) + .filter(exitEpoch -> !exitEpoch.equals(FAR_FUTURE_EPOCH)) + .max(UInt64::compareTo) + .orElse(UInt64.ZERO); + return maxExitEpochFromValidatorSet.max(activationExitEpoch).increment(); } } diff --git a/ethereum/spec/src/test/java/tech/pegasys/teku/spec/logic/versions/electra/forktransition/ElectraStateUpgradeTest.java b/ethereum/spec/src/test/java/tech/pegasys/teku/spec/logic/versions/electra/forktransition/ElectraStateUpgradeTest.java index a4b1f00a64d..716c20a1e40 100644 --- a/ethereum/spec/src/test/java/tech/pegasys/teku/spec/logic/versions/electra/forktransition/ElectraStateUpgradeTest.java +++ b/ethereum/spec/src/test/java/tech/pegasys/teku/spec/logic/versions/electra/forktransition/ElectraStateUpgradeTest.java @@ -78,7 +78,10 @@ void canUpgradeFromDeneb() { // = (64 *10^9) - (64 *10^9) MOD 10^9 // = (64 *10^9) - 0 assertThat(post.getExitBalanceToConsume()).isEqualTo(UInt64.valueOf(64_000_000_000L)); - assertThat(post.getEarliestExitEpoch()).isEqualTo(slot.dividedBy(8).increment()); + assertThat(post.getEarliestExitEpoch()) + .isEqualTo( + miscHelpersElectra.computeActivationExitEpoch( + spec.computeEpochAtSlot(slot).increment())); assertThat(post.getConsolidationBalanceToConsume()).isEqualTo(UInt64.ZERO); // 80_000/8 (slots -> epochs) + max_seed_lookahead + 1 assertThat(post.getEarliestConsolidationEpoch()).isEqualTo(UInt64.valueOf(10005)); diff --git a/fuzz/src/test/java/tech/pegasys/teku/fuzz/FuzzUtilTest.java b/fuzz/src/test/java/tech/pegasys/teku/fuzz/FuzzUtilTest.java index 05811285b46..1cae01840a6 100644 --- a/fuzz/src/test/java/tech/pegasys/teku/fuzz/FuzzUtilTest.java +++ b/fuzz/src/test/java/tech/pegasys/teku/fuzz/FuzzUtilTest.java @@ -22,6 +22,7 @@ import java.util.Optional; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.junit.BouncyCastleExtension; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.xerial.snappy.Snappy; @@ -376,6 +377,8 @@ public void fuzzWithdrawalRequest_minimal() { assertThat(result.get()).isEqualTo(expected); } + // TODO fix as part of https://github.com/Consensys/teku/pull/8876 + @Disabled("Disabling until we have a fix for this") @Test public void fuzzConsolidationRequest_minimal() { final FuzzUtil fuzzUtil = new FuzzUtil(false, true); From b4af97bbdbb11238f70de7329a117b874d3273a9 Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Fri, 6 Dec 2024 16:39:05 +0100 Subject: [PATCH 14/20] Missing synchronized method (#8894) --- .../util/BlockBlobSidecarsTrackersPoolImpl.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/util/BlockBlobSidecarsTrackersPoolImpl.java b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/util/BlockBlobSidecarsTrackersPoolImpl.java index d305fb55fa8..b63d8894097 100644 --- a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/util/BlockBlobSidecarsTrackersPoolImpl.java +++ b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/util/BlockBlobSidecarsTrackersPoolImpl.java @@ -407,7 +407,8 @@ public synchronized boolean containsBlobSidecar(final BlobIdentifier blobIdentif } @Override - public Optional getBlobSidecar(final Bytes32 blockRoot, final UInt64 index) { + public synchronized Optional getBlobSidecar( + final Bytes32 blockRoot, final UInt64 index) { return Optional.ofNullable(blockBlobSidecarsTrackers.get(blockRoot)) .flatMap(tracker -> tracker.getBlobSidecar(index)); } From 1b68e2b6bac7f4a5391e0eae8a41a4714bb9d36f Mon Sep 17 00:00:00 2001 From: Stefan Bratanov Date: Fri, 6 Dec 2024 16:46:36 +0000 Subject: [PATCH 15/20] Fix `is_syncing` API reporting when head is synced optimistically (#8889) --- CHANGELOG.md | 1 + .../beaconrestapi/handlers/v1/node/GetSyncing.java | 10 +++++----- .../handlers/v1/node/GetSyncingTest.java | 11 +++++++++++ 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d868630e8f9..6e045d3d29c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,3 +15,4 @@ ### Bug Fixes - Added a startup script for unix systems to ensure that when jemalloc is installed the script sets the LD_PRELOAD environment variable to the use the jemalloc library +- Set `is_syncing` to `false` instead of `true` for the `/eth/v1/node/syncing` API endpoint when the head is optimistic and the sync distance is 0 diff --git a/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/node/GetSyncing.java b/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/node/GetSyncing.java index 749346dd4de..ba1851c670d 100644 --- a/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/node/GetSyncing.java +++ b/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/node/GetSyncing.java @@ -97,12 +97,11 @@ public SyncStatusData( final ExecutionClientDataProvider executionClientDataProvider) { final SyncingStatus status = syncProvider.getSyncingStatus(); final SyncState syncState = syncProvider.getCurrentSyncState(); - this.isSyncing = !syncState.isInSync(); + this.slotsBehind = calculateSlotsBehind(status, syncState); + this.isSyncing = !slotsBehind.isZero(); this.elOffline = Optional.of(!executionClientDataProvider.isExecutionClientAvailable()); this.isOptimistic = Optional.of(syncState.isOptimistic()); this.currentSlot = status.getCurrentSlot(); - // do this last, after isSyncing is calculated - this.slotsBehind = calculateSlotsBehind(status); } SyncStatusData( @@ -138,8 +137,9 @@ public UInt64 getSlotsBehind() { return slotsBehind; } - private UInt64 calculateSlotsBehind(final SyncingStatus syncingStatus) { - if (isSyncing && syncingStatus.getHighestSlot().isPresent()) { + private UInt64 calculateSlotsBehind( + final SyncingStatus syncingStatus, final SyncState syncState) { + if (!syncState.isInSync() && syncingStatus.getHighestSlot().isPresent()) { final UInt64 highestSlot = syncingStatus.getHighestSlot().get(); return highestSlot.minusMinZero(syncingStatus.getCurrentSlot()); } diff --git a/data/beaconrestapi/src/test/java/tech/pegasys/teku/beaconrestapi/handlers/v1/node/GetSyncingTest.java b/data/beaconrestapi/src/test/java/tech/pegasys/teku/beaconrestapi/handlers/v1/node/GetSyncingTest.java index d3d99f657b5..40d8ffe7432 100644 --- a/data/beaconrestapi/src/test/java/tech/pegasys/teku/beaconrestapi/handlers/v1/node/GetSyncingTest.java +++ b/data/beaconrestapi/src/test/java/tech/pegasys/teku/beaconrestapi/handlers/v1/node/GetSyncingTest.java @@ -57,6 +57,17 @@ public void shouldGetSyncStatusInSync() throws Exception { .isEqualTo(new GetSyncing.SyncStatusData(false, false, false, 10, 0)); } + @Test + public void shouldGetSyncStatusInSyncWhenHeadIsOptimistic() throws Exception { + when(syncService.getSyncStatus()).thenReturn(getSyncStatus(false, 1, 10, 10)); + when(syncService.getCurrentSyncState()).thenReturn(SyncState.OPTIMISTIC_SYNCING); + + handler.handleRequest(request); + assertThat(request.getResponseCode()).isEqualTo(SC_OK); + assertThat(request.getResponseBody()) + .isEqualTo(new GetSyncing.SyncStatusData(false, true, false, 10, 0)); + } + @Test public void shouldGetElOffline() throws Exception { when(syncService.getSyncStatus()).thenReturn(getSyncStatus(false, 1, 10, 11)); From 16fcabd94516fece3376548a95b958caa820c538 Mon Sep 17 00:00:00 2001 From: Lucas Saldanha Date: Mon, 9 Dec 2024 16:53:21 +1300 Subject: [PATCH 16/20] Dependency updates December (#8898) --- gradle/versions.gradle | 38 +++++++++---------- gradle/wrapper/gradle-wrapper.properties | 4 +- .../restapi/SwaggerUIBuilder.java | 2 +- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/gradle/versions.gradle b/gradle/versions.gradle index b49b0042dde..4a104e6ec8e 100644 --- a/gradle/versions.gradle +++ b/gradle/versions.gradle @@ -1,11 +1,11 @@ dependencyManagement { dependencies { - dependency 'com.fasterxml.jackson.core:jackson-databind:2.18.1' - dependency 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.18.1' - dependency 'com.fasterxml.jackson.dataformat:jackson-dataformat-toml:2.18.1' - dependency 'com.fasterxml.jackson.module:jackson-module-kotlin:2.18.1' + dependency 'com.fasterxml.jackson.core:jackson-databind:2.18.2' + dependency 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.18.2' + dependency 'com.fasterxml.jackson.dataformat:jackson-dataformat-toml:2.18.2' + dependency 'com.fasterxml.jackson.module:jackson-module-kotlin:2.18.2' - dependencySet(group: 'com.google.errorprone', version: '2.35.1') { + dependencySet(group: 'com.google.errorprone', version: '2.36.0') { entry 'error_prone_annotation' entry 'error_prone_check_api' entry 'error_prone_core' @@ -16,7 +16,7 @@ dependencyManagement { dependency 'com.google.guava:guava:33.1.0-jre' - dependency 'org.jsoup:jsoup:1.18.1' + dependency 'org.jsoup:jsoup:1.18.3' dependency 'com.launchdarkly:okhttp-eventsource:4.1.1' @@ -38,11 +38,11 @@ dependencyManagement { dependency 'org.hdrhistogram:HdrHistogram:2.2.2' - dependency 'org.jetbrains.kotlin:kotlin-stdlib:2.0.21' + dependency 'org.jetbrains.kotlin:kotlin-stdlib:2.1.0' dependency 'org.mock-server:mockserver-junit-jupiter:5.15.0' - dependencySet(group: 'io.swagger.core.v3', version: '2.2.25') { + dependencySet(group: 'io.swagger.core.v3', version: '2.2.26') { entry 'swagger-parser' entry 'swagger-core' entry 'swagger-models' @@ -50,10 +50,10 @@ dependencyManagement { } // On update don't forget to change version in tech.pegasys.teku.infrastructure.restapi.SwaggerUIBuilder - dependency 'org.webjars:swagger-ui:5.17.14' + dependency 'org.webjars:swagger-ui:5.18.2' dependency 'org.thymeleaf:thymeleaf:3.1.2.RELEASE' - dependency 'io.github.classgraph:classgraph:4.8.177' + dependency 'io.github.classgraph:classgraph:4.8.179' dependencySet(group: 'com.github.oshi', version: '6.6.5') { entry 'oshi-core' entry 'oshi-core-java11' @@ -64,13 +64,13 @@ dependencyManagement { entry 'netty-codec-http' } - dependencySet(group: 'io.vertx', version: '4.5.10') { + dependencySet(group: 'io.vertx', version: '4.5.11') { entry 'vertx-codegen' entry 'vertx-core' entry 'vertx-unit' entry 'vertx-web' } - dependency 'io.projectreactor:reactor-core:3.6.11' + dependency 'io.projectreactor:reactor-core:3.7.0' dependency 'it.unimi.dsi:fastutil:8.5.15' @@ -86,10 +86,10 @@ dependencyManagement { dependency 'org.apache.commons:commons-text:1.12.0' dependency 'org.apache.commons:commons-lang3:3.17.0' - dependency 'commons-io:commons-io:2.17.0' + dependency 'commons-io:commons-io:2.18.0' dependency 'org.commonjava.mimeparse:mimeparse:0.1.3.3' - dependencySet(group: 'org.apache.logging.log4j', version: '2.24.1') { + dependencySet(group: 'org.apache.logging.log4j', version: '2.24.2') { entry 'log4j-api' entry 'log4j-core' entry 'log4j-slf4j-impl' @@ -124,8 +124,8 @@ dependencyManagement { entry 'jmh-core' entry 'jmh-generator-annprocess' } - dependency 'org.quartz-scheduler:quartz:2.3.2' - dependency 'org.rocksdb:rocksdbjni:9.5.2' + dependency 'org.quartz-scheduler:quartz:2.5.0' + dependency 'org.rocksdb:rocksdbjni:9.7.3' dependency 'org.fusesource.leveldbjni:leveldbjni-win64:1.8' dependency 'org.fusesource.leveldbjni:leveldbjni-win32:1.8' dependency 'tech.pegasys:leveldb-native:0.3.1' @@ -152,14 +152,14 @@ dependencyManagement { entry('plugin-api') } - dependencySet(group: 'org.testcontainers', version: '1.20.3') { + dependencySet(group: 'org.testcontainers', version: '1.20.4') { entry "testcontainers" entry "junit-jupiter" } // discovery includes tuweni libraries under a different name so version resolution doesn't work // exclude them here and leave them to be included on the classpath by the version we use - dependency('tech.pegasys.discovery:discovery:24.9.1') { + dependency('tech.pegasys.discovery:discovery:24.12.0') { exclude 'org.apache.tuweni:bytes' exclude 'org.apache.tuweni:crypto' exclude 'org.apache.tuweni:units' @@ -176,6 +176,6 @@ dependencyManagement { entry 'jjwt-jackson' } - dependency 'net.jqwik:jqwik:1.9.1' + dependency 'net.jqwik:jqwik:1.9.2' } } diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index fb602ee2af0..eb1a55be0e1 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,7 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionSha256Sum=31c55713e40233a8303827ceb42ca48a47267a0ad4bab9177123121e71524c26 -distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.2-bin.zip +distributionSha256Sum=f397b287023acdba1e9f6fc5ea72d22dd63669d59ed4a289a29b1a76eee151c6 +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/infrastructure/restapi/src/main/java/tech/pegasys/teku/infrastructure/restapi/SwaggerUIBuilder.java b/infrastructure/restapi/src/main/java/tech/pegasys/teku/infrastructure/restapi/SwaggerUIBuilder.java index a1194120dc6..c636fd9f426 100644 --- a/infrastructure/restapi/src/main/java/tech/pegasys/teku/infrastructure/restapi/SwaggerUIBuilder.java +++ b/infrastructure/restapi/src/main/java/tech/pegasys/teku/infrastructure/restapi/SwaggerUIBuilder.java @@ -30,7 +30,7 @@ public class SwaggerUIBuilder { // Version here MUST match `swagger-ui` library version - private static final String SWAGGER_UI_VERSION = "5.17.14"; + private static final String SWAGGER_UI_VERSION = "5.18.2"; private static final String SWAGGER_UI_PATH = "/swagger-ui"; private static final String SWAGGER_HOSTED_PATH = "/webjars/swagger-ui/" + SWAGGER_UI_VERSION; From 4ac301843b5c330be0fd6715fb6d8b67c1e2388e Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Mon, 9 Dec 2024 10:18:13 +0100 Subject: [PATCH 17/20] fix gossip subscriptions (#8896) --- .../gossip/topics/Eth2GossipTopicFilter.java | 10 ++- .../topics/Eth2GossipTopicFilterTest.java | 87 +++++++++++-------- 2 files changed, 60 insertions(+), 37 deletions(-) diff --git a/networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/gossip/topics/Eth2GossipTopicFilter.java b/networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/gossip/topics/Eth2GossipTopicFilter.java index 1aa08b34474..d1209d1e0ec 100644 --- a/networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/gossip/topics/Eth2GossipTopicFilter.java +++ b/networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/gossip/topics/Eth2GossipTopicFilter.java @@ -25,6 +25,7 @@ import tech.pegasys.teku.networking.p2p.libp2p.gossip.GossipTopicFilter; import tech.pegasys.teku.spec.Spec; import tech.pegasys.teku.spec.SpecMilestone; +import tech.pegasys.teku.spec.SpecVersion; import tech.pegasys.teku.spec.datastructures.state.ForkInfo; import tech.pegasys.teku.storage.client.RecentChainData; @@ -57,15 +58,18 @@ private Set computeRelevantTopics( recentChainData.getMilestoneByForkDigest(forkDigest).orElseThrow(); final Set topics = getAllTopics(gossipEncoding, forkDigest, spec, specMilestone); spec.getForkSchedule().getForks().stream() - .filter(fork -> fork.getEpoch().isGreaterThanOrEqualTo(forkInfo.getFork().getEpoch())) + .filter(fork -> fork.getEpoch().isGreaterThan(forkInfo.getFork().getEpoch())) .forEach( futureFork -> { + final SpecVersion futureSpecVersion = spec.atEpoch(futureFork.getEpoch()); final Bytes4 futureForkDigest = - spec.atEpoch(futureFork.getEpoch()) + futureSpecVersion .miscHelpers() .computeForkDigest( futureFork.getCurrentVersion(), forkInfo.getGenesisValidatorsRoot()); - topics.addAll(getAllTopics(gossipEncoding, futureForkDigest, spec, specMilestone)); + topics.addAll( + getAllTopics( + gossipEncoding, futureForkDigest, spec, futureSpecVersion.getMilestone())); }); return topics; } diff --git a/networking/eth2/src/test/java/tech/pegasys/teku/networking/eth2/gossip/topics/Eth2GossipTopicFilterTest.java b/networking/eth2/src/test/java/tech/pegasys/teku/networking/eth2/gossip/topics/Eth2GossipTopicFilterTest.java index 2eac5f144db..b32868eef90 100644 --- a/networking/eth2/src/test/java/tech/pegasys/teku/networking/eth2/gossip/topics/Eth2GossipTopicFilterTest.java +++ b/networking/eth2/src/test/java/tech/pegasys/teku/networking/eth2/gossip/topics/Eth2GossipTopicFilterTest.java @@ -14,18 +14,20 @@ package tech.pegasys.teku.networking.eth2.gossip.topics; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assumptions.assumeThat; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.spy; import static tech.pegasys.teku.networking.eth2.gossip.encoding.GossipEncoding.SSZ_SNAPPY; import static tech.pegasys.teku.networking.eth2.gossip.topics.GossipTopicName.getAttestationSubnetTopicName; import static tech.pegasys.teku.networking.eth2.gossip.topics.GossipTopicName.getBlobSidecarSubnetTopicName; import static tech.pegasys.teku.networking.eth2.gossip.topics.GossipTopicName.getSyncCommitteeSubnetTopicName; +import static tech.pegasys.teku.spec.SpecMilestone.DENEB; +import static tech.pegasys.teku.spec.SpecMilestone.ELECTRA; import static tech.pegasys.teku.spec.constants.NetworkConstants.SYNC_COMMITTEE_SUBNET_COUNT; import java.util.List; import java.util.Optional; -import org.apache.tuweni.bytes.Bytes32; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.TestTemplate; import tech.pegasys.teku.infrastructure.bytes.Bytes4; @@ -34,58 +36,60 @@ import tech.pegasys.teku.spec.SpecMilestone; import tech.pegasys.teku.spec.TestSpecContext; import tech.pegasys.teku.spec.TestSpecFactory; -import tech.pegasys.teku.spec.TestSpecInvocationContextProvider; +import tech.pegasys.teku.spec.TestSpecInvocationContextProvider.SpecContext; import tech.pegasys.teku.spec.config.SpecConfig; import tech.pegasys.teku.spec.config.SpecConfigDeneb; import tech.pegasys.teku.spec.datastructures.state.Fork; import tech.pegasys.teku.spec.datastructures.state.ForkInfo; -import tech.pegasys.teku.spec.util.DataStructureUtil; import tech.pegasys.teku.storage.client.RecentChainData; +import tech.pegasys.teku.storage.storageSystem.InMemoryStorageSystemBuilder; +import tech.pegasys.teku.storage.storageSystem.StorageSystem; -@TestSpecContext(milestone = {SpecMilestone.DENEB, SpecMilestone.ELECTRA}) +@TestSpecContext(milestone = {DENEB, ELECTRA}) class Eth2GossipTopicFilterTest { - - private final RecentChainData recentChainData = mock(RecentChainData.class); - private final UInt64 currentForkEpoch = UInt64.valueOf(10); + private final UInt64 nextMilestoneForkEpoch = UInt64.valueOf(10); + private RecentChainData recentChainData; private Spec spec; - private SpecMilestone specMilestone; - private DataStructureUtil dataStructureUtil; - private ForkInfo forkInfo; + private SpecMilestone currentSpecMilestone; + private SpecMilestone nextSpecMilestone; + private ForkInfo currentForkInfo; private Eth2GossipTopicFilter filter; - private Bytes4 currentForkDigest; private Bytes4 nextForkDigest; @BeforeEach - void setUp(final TestSpecInvocationContextProvider.SpecContext specContext) { - specMilestone = specContext.getSpecMilestone(); + void setUp(final SpecContext specContext) { + // we set up a spec that will be transitioning to specContext.getSpecMilestone() in + // currentForkEpoch epochs + // current milestone is actually the previous milestone + currentSpecMilestone = specContext.getSpecMilestone().getPreviousMilestone(); + nextSpecMilestone = specContext.getSpecMilestone(); spec = - switch (specContext.getSpecMilestone()) { + switch (nextSpecMilestone) { case PHASE0 -> throw new IllegalArgumentException("Phase0 is an unsupported milestone"); case ALTAIR -> throw new IllegalArgumentException("Altair is an unsupported milestone"); case BELLATRIX -> throw new IllegalArgumentException("Bellatrix is an unsupported milestone"); case CAPELLA -> throw new IllegalArgumentException("Capella is an unsupported milestone"); - case DENEB -> TestSpecFactory.createMinimalWithDenebForkEpoch(currentForkEpoch); - case ELECTRA -> TestSpecFactory.createMinimalWithElectraForkEpoch(currentForkEpoch); + case DENEB -> TestSpecFactory.createMinimalWithDenebForkEpoch(nextMilestoneForkEpoch); + case ELECTRA -> TestSpecFactory.createMinimalWithElectraForkEpoch(nextMilestoneForkEpoch); }; - dataStructureUtil = new DataStructureUtil(spec); + + final StorageSystem storageSystem = InMemoryStorageSystemBuilder.buildDefault(spec); + storageSystem.chainUpdater().initializeGenesis(); + + recentChainData = spy(storageSystem.recentChainData()); filter = new Eth2GossipTopicFilter(recentChainData, SSZ_SNAPPY, spec); - final Bytes32 genesisValidatorsRoot = dataStructureUtil.randomBytes32(); final List forks = spec.getForkSchedule().getForks(); - forkInfo = new ForkInfo(forks.get(0), genesisValidatorsRoot); - currentForkDigest = forkInfo.getForkDigest(spec); + currentForkInfo = recentChainData.getCurrentForkInfo().orElseThrow(); final Fork nextFork = forks.get(1); nextForkDigest = spec.atEpoch(nextFork.getEpoch()) .miscHelpers() - .computeForkDigest(nextFork.getCurrentVersion(), genesisValidatorsRoot); - - when(recentChainData.getCurrentForkInfo()).thenReturn(Optional.of(forkInfo)); - when(recentChainData.getNextFork(forkInfo.getFork())).thenReturn(Optional.of(nextFork)); - when(recentChainData.getMilestoneByForkDigest(currentForkDigest)) - .thenReturn(Optional.of(specMilestone)); + .computeForkDigest( + nextFork.getCurrentVersion(), + recentChainData.getGenesisData().orElseThrow().getGenesisValidatorsRoot()); } @TestTemplate @@ -95,7 +99,7 @@ void shouldNotAllowIrrelevantTopics() { @TestTemplate void shouldNotRequireNextForkToBePresent() { - when(recentChainData.getNextFork(any())).thenReturn(Optional.empty()); + doAnswer(invocation -> Optional.empty()).when(recentChainData).getNextFork(any()); assertThat(filter.isRelevantTopic(getTopicName(GossipTopicName.BEACON_BLOCK))).isTrue(); } @@ -129,18 +133,33 @@ void shouldConsiderAllSyncCommitteeSubnetsRelevant() { } @TestTemplate - void shouldConsiderAllBlobSidecarSubnetsRelevant() { - final SpecConfig config = spec.forMilestone(SpecMilestone.DENEB).getConfig(); + void shouldConsiderAllBlobSidecarSubnetsRelevantForCurrentMilestone() { + final SpecConfig config = spec.forMilestone(currentSpecMilestone).getConfig(); + assumeThat(config.toVersionDeneb()).isPresent(); final SpecConfigDeneb specConfigDeneb = SpecConfigDeneb.required(config); for (int i = 0; i < specConfigDeneb.getBlobSidecarSubnetCount(); i++) { assertThat(filter.isRelevantTopic(getTopicName(getBlobSidecarSubnetTopicName(i)))).isTrue(); } } + @TestTemplate + void shouldConsiderAllBlobSidecarSubnetsRelevantForNextMilestone() { + final SpecConfig config = spec.forMilestone(nextSpecMilestone).getConfig(); + assumeThat(config.toVersionDeneb()).isPresent(); + final SpecConfigDeneb specConfigDeneb = SpecConfigDeneb.required(config); + for (int i = 0; i < specConfigDeneb.getBlobSidecarSubnetCount(); i++) { + assertThat(filter.isRelevantTopic(getNextForkTopicName(getBlobSidecarSubnetTopicName(i)))) + .isTrue(); + } + } + @TestTemplate void shouldNotConsiderBlobSidecarWithIncorrectSubnetIdRelevant() { final int blobSidecarSubnetCount = - SpecConfigDeneb.required(spec.forMilestone(specMilestone).getConfig()) + spec.forMilestone(nextSpecMilestone) + .getConfig() + .toVersionDeneb() + .orElseThrow() .getBlobSidecarSubnetCount(); assertThat( filter.isRelevantTopic( @@ -162,11 +181,11 @@ void shouldNotAllowTopicsWithUnknownForkDigest() { } private String getTopicName(final GossipTopicName name) { - return GossipTopics.getTopic(forkInfo.getForkDigest(spec), name, SSZ_SNAPPY); + return GossipTopics.getTopic(currentForkInfo.getForkDigest(spec), name, SSZ_SNAPPY); } private String getTopicName(final String name) { - return GossipTopics.getTopic(forkInfo.getForkDigest(spec), name, SSZ_SNAPPY); + return GossipTopics.getTopic(currentForkInfo.getForkDigest(spec), name, SSZ_SNAPPY); } private String getNextForkTopicName(final GossipTopicName name) { From bf649d00822a9868d9000328541c98c615d24f7a Mon Sep 17 00:00:00 2001 From: Paul Harris Date: Mon, 9 Dec 2024 23:26:23 +1000 Subject: [PATCH 18/20] update post attestation interface for electra (#8893) --- ...tAttestationsV2ElectraIntegrationTest.java | 79 +++++++++++++ .../PostAttestationsV2IntegrationTest.java | 104 +++++++++--------- .../_eth_v2_beacon_pool_attestations.json | 2 +- .../beacon/schema/SingleAttestation.json | 28 +++++ .../v2/beacon/PostAttestationsV2.java | 63 ++++++++--- .../postAttestationRequestBodyELECTRA.json | 28 ++--- 6 files changed, 223 insertions(+), 81 deletions(-) create mode 100644 data/beaconrestapi/src/integration-test/java/tech/pegasys/teku/beaconrestapi/v2/beacon/PostAttestationsV2ElectraIntegrationTest.java create mode 100644 data/beaconrestapi/src/integration-test/resources/tech/pegasys/teku/beaconrestapi/beacon/schema/SingleAttestation.json diff --git a/data/beaconrestapi/src/integration-test/java/tech/pegasys/teku/beaconrestapi/v2/beacon/PostAttestationsV2ElectraIntegrationTest.java b/data/beaconrestapi/src/integration-test/java/tech/pegasys/teku/beaconrestapi/v2/beacon/PostAttestationsV2ElectraIntegrationTest.java new file mode 100644 index 00000000000..687feebd189 --- /dev/null +++ b/data/beaconrestapi/src/integration-test/java/tech/pegasys/teku/beaconrestapi/v2/beacon/PostAttestationsV2ElectraIntegrationTest.java @@ -0,0 +1,79 @@ +/* + * Copyright Consensys Software Inc., 2024 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.beaconrestapi.v2.beacon; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import okhttp3.Response; +import org.junit.jupiter.api.BeforeEach; +import tech.pegasys.teku.beaconrestapi.handlers.v2.beacon.PostAttestationsV2; +import tech.pegasys.teku.infrastructure.json.JsonUtil; +import tech.pegasys.teku.infrastructure.json.types.SerializableTypeDefinition; +import tech.pegasys.teku.spec.SpecMilestone; +import tech.pegasys.teku.spec.TestSpecFactory; +import tech.pegasys.teku.spec.datastructures.operations.Attestation; +import tech.pegasys.teku.spec.datastructures.operations.SingleAttestation; +import tech.pegasys.teku.spec.schemas.SchemaDefinitionsElectra; +import tech.pegasys.teku.spec.util.DataStructureUtil; + +public class PostAttestationsV2ElectraIntegrationTest extends PostAttestationsV2IntegrationTest { + protected SerializableTypeDefinition> attestationsListTypeDef; + + @Override + @BeforeEach + void setup() { + spec = TestSpecFactory.createMinimalElectra(); + specMilestone = SpecMilestone.ELECTRA; + startRestAPIAtGenesis(specMilestone); + dataStructureUtil = new DataStructureUtil(spec); + attestationsListTypeDef = + SerializableTypeDefinition.listOf( + SchemaDefinitionsElectra.required(spec.getGenesisSchemaDefinitions()) + .getSingleAttestationSchema() + .getJsonTypeDefinition()); + } + + @Override + protected List getAttestationList(final int listSize) { + final List attestations = new ArrayList<>(listSize); + for (int i = 0; i < listSize; i++) { + attestations.add(dataStructureUtil.randomSingleAttestation()); + } + return attestations; + } + + @Override + @SuppressWarnings("unchecked") + protected Response postAttestations(final List attestations, final String milestone) + throws IOException { + final SerializableTypeDefinition> attestationsListTypeDef = + SerializableTypeDefinition.listOf( + SchemaDefinitionsElectra.required(spec.getGenesisSchemaDefinitions()) + .getSingleAttestationSchema() + .getJsonTypeDefinition()); + if (milestone == null) { + return post( + PostAttestationsV2.ROUTE, + JsonUtil.serialize((List) attestations, attestationsListTypeDef)); + } + return post( + PostAttestationsV2.ROUTE, + JsonUtil.serialize((List) attestations, attestationsListTypeDef), + Collections.emptyMap(), + Optional.of(milestone)); + } +} diff --git a/data/beaconrestapi/src/integration-test/java/tech/pegasys/teku/beaconrestapi/v2/beacon/PostAttestationsV2IntegrationTest.java b/data/beaconrestapi/src/integration-test/java/tech/pegasys/teku/beaconrestapi/v2/beacon/PostAttestationsV2IntegrationTest.java index f468e7651d1..8a4d9b35036 100644 --- a/data/beaconrestapi/src/integration-test/java/tech/pegasys/teku/beaconrestapi/v2/beacon/PostAttestationsV2IntegrationTest.java +++ b/data/beaconrestapi/src/integration-test/java/tech/pegasys/teku/beaconrestapi/v2/beacon/PostAttestationsV2IntegrationTest.java @@ -20,13 +20,14 @@ import static tech.pegasys.teku.infrastructure.http.RestApiConstants.HEADER_CONSENSUS_VERSION; import com.fasterxml.jackson.databind.JsonNode; +import java.io.IOException; +import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Locale; import java.util.Optional; import okhttp3.Response; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.TestTemplate; +import org.junit.jupiter.api.Test; import tech.pegasys.teku.beaconrestapi.AbstractDataBackedRestAPIIntegrationTest; import tech.pegasys.teku.beaconrestapi.handlers.v2.beacon.PostAttestationsV2; import tech.pegasys.teku.infrastructure.async.SafeFuture; @@ -35,75 +36,51 @@ import tech.pegasys.teku.infrastructure.json.types.SerializableTypeDefinition; import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.SpecMilestone; -import tech.pegasys.teku.spec.TestSpecContext; -import tech.pegasys.teku.spec.TestSpecInvocationContextProvider; +import tech.pegasys.teku.spec.TestSpecFactory; import tech.pegasys.teku.spec.datastructures.operations.Attestation; import tech.pegasys.teku.spec.util.DataStructureUtil; import tech.pegasys.teku.validator.api.SubmitDataError; -@TestSpecContext(milestone = {SpecMilestone.PHASE0, SpecMilestone.ELECTRA}) public class PostAttestationsV2IntegrationTest extends AbstractDataBackedRestAPIIntegrationTest { - private DataStructureUtil dataStructureUtil; - private SpecMilestone specMilestone; - private SerializableTypeDefinition> attestationsListTypeDef; + protected DataStructureUtil dataStructureUtil; + protected SpecMilestone specMilestone; @BeforeEach - void setup(final TestSpecInvocationContextProvider.SpecContext specContext) { - spec = specContext.getSpec(); - specMilestone = specContext.getSpecMilestone(); + void setup() { + spec = TestSpecFactory.createMinimalPhase0(); + specMilestone = SpecMilestone.PHASE0; startRestAPIAtGenesis(specMilestone); - dataStructureUtil = specContext.getDataStructureUtil(); - attestationsListTypeDef = - SerializableTypeDefinition.listOf( - spec.getGenesisSchemaDefinitions() - .getAttestationSchema() - .castTypeToAttestationSchema() - .getJsonTypeDefinition()); + dataStructureUtil = new DataStructureUtil(spec); } - @TestTemplate + @Test void shouldPostAttestations_NoErrors() throws Exception { - final List attestations = - List.of(dataStructureUtil.randomAttestation(), dataStructureUtil.randomAttestation()); + final List attestations = getAttestationList(2); when(validatorApiChannel.sendSignedAttestations(attestations)) .thenReturn(SafeFuture.completedFuture(Collections.emptyList())); - final Response response = - post( - PostAttestationsV2.ROUTE, - JsonUtil.serialize(attestations, attestationsListTypeDef), - Collections.emptyMap(), - Optional.of(specMilestone.name().toLowerCase(Locale.ROOT))); + final Response response = postAttestations(attestations, specMilestone.name()); assertThat(response.code()).isEqualTo(SC_OK); assertThat(response.body().string()).isEmpty(); } - @TestTemplate + @Test void shouldPartiallyPostAttestations_ReturnsErrors() throws Exception { final SubmitDataError firstSubmitDataError = new SubmitDataError(UInt64.ZERO, "Bad attestation"); final SubmitDataError secondSubmitDataError = new SubmitDataError(UInt64.ONE, "Very bad attestation"); - final List attestations = - List.of( - dataStructureUtil.randomAttestation(), - dataStructureUtil.randomAttestation(), - dataStructureUtil.randomAttestation()); + final List attestations = getAttestationList(3); when(validatorApiChannel.sendSignedAttestations(attestations)) .thenReturn( SafeFuture.completedFuture(List.of(firstSubmitDataError, secondSubmitDataError))); - final Response response = - post( - PostAttestationsV2.ROUTE, - JsonUtil.serialize(attestations, attestationsListTypeDef), - Collections.emptyMap(), - Optional.of(specMilestone.name().toLowerCase(Locale.ROOT))); + final Response response = postAttestations(attestations, specMilestone.name()); assertThat(response.code()).isEqualTo(SC_BAD_REQUEST); final JsonNode resultAsJsonNode = JsonTestUtil.parseAsJsonNode(response.body().string()); @@ -121,16 +98,14 @@ void shouldPartiallyPostAttestations_ReturnsErrors() throws Exception { .isEqualTo(secondSubmitDataError.message()); } - @TestTemplate + @Test void shouldFailWhenMissingConsensusHeader() throws Exception { - final List attestations = - List.of(dataStructureUtil.randomAttestation(), dataStructureUtil.randomAttestation()); + final List attestations = getAttestationList(2); when(validatorApiChannel.sendSignedAttestations(attestations)) .thenReturn(SafeFuture.completedFuture(Collections.emptyList())); - final Response response = - post(PostAttestationsV2.ROUTE, JsonUtil.serialize(attestations, attestationsListTypeDef)); + final Response response = postAttestations(attestations, null); assertThat(response.code()).isEqualTo(SC_BAD_REQUEST); @@ -139,20 +114,14 @@ void shouldFailWhenMissingConsensusHeader() throws Exception { .isEqualTo("Missing required header value for (%s)", HEADER_CONSENSUS_VERSION); } - @TestTemplate + @Test void shouldFailWhenBadConsensusHeaderValue() throws Exception { - final List attestations = - List.of(dataStructureUtil.randomAttestation(), dataStructureUtil.randomAttestation()); + final List attestations = getAttestationList(2); when(validatorApiChannel.sendSignedAttestations(attestations)) .thenReturn(SafeFuture.completedFuture(Collections.emptyList())); final String badConsensusHeaderValue = "NonExistingMileStone"; - final Response response = - post( - PostAttestationsV2.ROUTE, - JsonUtil.serialize(attestations, attestationsListTypeDef), - Collections.emptyMap(), - Optional.of(badConsensusHeaderValue)); + final Response response = postAttestations(attestations, badConsensusHeaderValue); assertThat(response.code()).isEqualTo(SC_BAD_REQUEST); @@ -163,4 +132,33 @@ void shouldFailWhenBadConsensusHeaderValue() throws Exception { "Invalid value for (%s) header: %s", HEADER_CONSENSUS_VERSION, badConsensusHeaderValue)); } + + protected List getAttestationList(final int listSize) { + final List attestations = new ArrayList<>(listSize); + for (int i = 0; i < listSize; i++) { + attestations.add(dataStructureUtil.randomAttestation()); + } + return attestations; + } + + @SuppressWarnings("unchecked") + protected Response postAttestations(final List attestations, final String milestone) + throws IOException { + final SerializableTypeDefinition> attestationsListTypeDef = + SerializableTypeDefinition.listOf( + spec.getGenesisSchemaDefinitions() + .getAttestationSchema() + .castTypeToAttestationSchema() + .getJsonTypeDefinition()); + if (milestone == null) { + return post( + PostAttestationsV2.ROUTE, + JsonUtil.serialize((List) attestations, attestationsListTypeDef)); + } + return post( + PostAttestationsV2.ROUTE, + JsonUtil.serialize((List) attestations, attestationsListTypeDef), + Collections.emptyMap(), + Optional.of(milestone)); + } } diff --git a/data/beaconrestapi/src/integration-test/resources/tech/pegasys/teku/beaconrestapi/beacon/paths/_eth_v2_beacon_pool_attestations.json b/data/beaconrestapi/src/integration-test/resources/tech/pegasys/teku/beaconrestapi/beacon/paths/_eth_v2_beacon_pool_attestations.json index 5f652893c81..7242e8ad9ee 100644 --- a/data/beaconrestapi/src/integration-test/resources/tech/pegasys/teku/beaconrestapi/beacon/paths/_eth_v2_beacon_pool_attestations.json +++ b/data/beaconrestapi/src/integration-test/resources/tech/pegasys/teku/beaconrestapi/beacon/paths/_eth_v2_beacon_pool_attestations.json @@ -100,7 +100,7 @@ "oneOf" : [ { "$ref" : "#/components/schemas/AttestationPhase0" }, { - "$ref" : "#/components/schemas/AttestationElectra" + "$ref" : "#/components/schemas/SingleAttestation" } ] } } diff --git a/data/beaconrestapi/src/integration-test/resources/tech/pegasys/teku/beaconrestapi/beacon/schema/SingleAttestation.json b/data/beaconrestapi/src/integration-test/resources/tech/pegasys/teku/beaconrestapi/beacon/schema/SingleAttestation.json new file mode 100644 index 00000000000..beb8224bfdc --- /dev/null +++ b/data/beaconrestapi/src/integration-test/resources/tech/pegasys/teku/beaconrestapi/beacon/schema/SingleAttestation.json @@ -0,0 +1,28 @@ +{ + "title" : "SingleAttestation", + "type" : "object", + "required" : [ "committee_index", "attester_index", "data", "signature" ], + "properties" : { + "committee_index" : { + "type" : "string", + "description" : "unsigned 64 bit integer", + "example" : "1", + "format" : "uint64" + }, + "attester_index" : { + "type" : "string", + "description" : "unsigned 64 bit integer", + "example" : "1", + "format" : "uint64" + }, + "data" : { + "$ref" : "#/components/schemas/AttestationData" + }, + "signature" : { + "type" : "string", + "pattern" : "^0x[a-fA-F0-9]{2,}$", + "description" : "SSZ hexadecimal", + "format" : "bytes" + } + } +} \ No newline at end of file diff --git a/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v2/beacon/PostAttestationsV2.java b/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v2/beacon/PostAttestationsV2.java index 2ddd488a0f0..f8e14f5edd1 100644 --- a/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v2/beacon/PostAttestationsV2.java +++ b/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v2/beacon/PostAttestationsV2.java @@ -15,22 +15,25 @@ import static tech.pegasys.teku.api.ValidatorDataProvider.PARTIAL_PUBLISH_FAILURE_MESSAGE; import static tech.pegasys.teku.beaconrestapi.BeaconRestApiTypes.ETH_CONSENSUS_VERSION_TYPE; -import static tech.pegasys.teku.beaconrestapi.handlers.v1.beacon.MilestoneDependentTypesUtil.getSchemaDefinitionForAllSupportedMilestones; -import static tech.pegasys.teku.beaconrestapi.handlers.v1.beacon.MilestoneDependentTypesUtil.headerBasedSelector; import static tech.pegasys.teku.infrastructure.http.HttpStatusCodes.SC_BAD_REQUEST; import static tech.pegasys.teku.infrastructure.http.HttpStatusCodes.SC_OK; +import static tech.pegasys.teku.infrastructure.http.RestApiConstants.HEADER_CONSENSUS_VERSION; import static tech.pegasys.teku.infrastructure.http.RestApiConstants.TAG_BEACON; import static tech.pegasys.teku.infrastructure.http.RestApiConstants.TAG_EXPERIMENTAL; import static tech.pegasys.teku.infrastructure.http.RestApiConstants.TAG_VALIDATOR_REQUIRED; import com.fasterxml.jackson.core.JsonProcessingException; import java.util.List; +import java.util.Map; import java.util.function.BiPredicate; import tech.pegasys.teku.api.DataProvider; import tech.pegasys.teku.api.ValidatorDataProvider; +import tech.pegasys.teku.api.exceptions.BadRequestException; import tech.pegasys.teku.beaconrestapi.schema.ErrorListBadRequest; import tech.pegasys.teku.infrastructure.async.SafeFuture; +import tech.pegasys.teku.infrastructure.json.types.DeserializableTypeDefinition; import tech.pegasys.teku.infrastructure.json.types.SerializableOneOfTypeDefinition; +import tech.pegasys.teku.infrastructure.json.types.SerializableOneOfTypeDefinitionBuilder; import tech.pegasys.teku.infrastructure.json.types.SerializableTypeDefinition; import tech.pegasys.teku.infrastructure.restapi.endpoints.AsyncApiResponse; import tech.pegasys.teku.infrastructure.restapi.endpoints.EndpointMetadata; @@ -40,7 +43,7 @@ import tech.pegasys.teku.spec.SpecMilestone; import tech.pegasys.teku.spec.datastructures.operations.Attestation; import tech.pegasys.teku.spec.schemas.SchemaDefinitionCache; -import tech.pegasys.teku.spec.schemas.SchemaDefinitions; +import tech.pegasys.teku.spec.schemas.SchemaDefinitionsElectra; import tech.pegasys.teku.validator.api.SubmitDataError; public class PostAttestationsV2 extends RestApiEndpoint { @@ -87,20 +90,27 @@ private static EndpointMetadata createMetadata( .milestoneAtSlot(attestation.getData().getSlot()) .equals(milestone); + final SerializableOneOfTypeDefinitionBuilder builder = + new SerializableOneOfTypeDefinitionBuilder().title("SignedAttestation"); + + builder.withType( + value -> attestationSchemaPredicate.test(value, SpecMilestone.PHASE0), + schemaDefinitionCache + .getSchemaDefinition(SpecMilestone.PHASE0) + .getAttestationSchema() + .getJsonTypeDefinition()); + builder.withType( + value -> attestationSchemaPredicate.test(value, SpecMilestone.ELECTRA), + SchemaDefinitionsElectra.required( + schemaDefinitionCache.getSchemaDefinition(SpecMilestone.ELECTRA)) + .getSingleAttestationSchema() + .getJsonTypeDefinition()); final SerializableOneOfTypeDefinition attestationSchemaDefinition = - getSchemaDefinitionForAllSupportedMilestones( - schemaDefinitionCache, - "SignedAttestation", - SchemaDefinitions::getAttestationSchema, - attestationSchemaPredicate); + builder.build(); final OneOfArrayJsonRequestContentTypeDefinition.BodyTypeSelector attestationBodySelector = - context -> - headerBasedSelector( - context.getHeaders(), - schemaDefinitionCache, - SchemaDefinitions::getAttestationSchema); + context -> headerBasedSelector(context.getHeaders(), schemaDefinitionCache); return EndpointMetadata.post(ROUTE) .operationId("submitPoolAttestationsV2") @@ -123,4 +133,31 @@ private static EndpointMetadata createMetadata( .withChainDataResponses() .build(); } + + public static DeserializableTypeDefinition headerBasedSelector( + final Map headers, final SchemaDefinitionCache schemaDefinitionCache) { + if (!headers.containsKey(HEADER_CONSENSUS_VERSION)) { + throw new BadRequestException( + String.format("Missing required header value for (%s)", HEADER_CONSENSUS_VERSION)); + } + try { + final SpecMilestone milestone = SpecMilestone.forName(headers.get(HEADER_CONSENSUS_VERSION)); + if (milestone.isLessThanOrEqualTo(SpecMilestone.DENEB)) { + return schemaDefinitionCache + .getSchemaDefinition(SpecMilestone.PHASE0) + .getAttestationSchema() + .getJsonTypeDefinition(); + } else { + return SchemaDefinitionsElectra.required( + schemaDefinitionCache.getSchemaDefinition(SpecMilestone.ELECTRA)) + .getSingleAttestationSchema() + .getJsonTypeDefinition(); + } + } catch (Exception e) { + throw new BadRequestException( + String.format( + "Invalid value for (%s) header: %s", + HEADER_CONSENSUS_VERSION, headers.get(HEADER_CONSENSUS_VERSION))); + } + } } diff --git a/data/beaconrestapi/src/test/resources/tech/pegasys/teku/beaconrestapi/handlers/v2/beacon/postAttestationRequestBodyELECTRA.json b/data/beaconrestapi/src/test/resources/tech/pegasys/teku/beaconrestapi/handlers/v2/beacon/postAttestationRequestBodyELECTRA.json index 1ec340afbd9..67ce9a3a449 100644 --- a/data/beaconrestapi/src/test/resources/tech/pegasys/teku/beaconrestapi/handlers/v2/beacon/postAttestationRequestBodyELECTRA.json +++ b/data/beaconrestapi/src/test/resources/tech/pegasys/teku/beaconrestapi/handlers/v2/beacon/postAttestationRequestBodyELECTRA.json @@ -1,20 +1,20 @@ [ { - "aggregation_bits": "0x4a9278690f62e1a353f1abf2b9701e13e8cdf4d9ac6b032ba43b05b25f713540ce24f3192819e752fb091217ff34b68e934a06d316b6060696f8f24749574c4b3ac2e4ccb6914c434b09a81fff523e12acbe299fcdad715593298d72ca70e1ffc743ad7ce89587fbb4b4c57db7856b7082db70ebddcbebe264f886236df2dd51539e10d4dcd5950e6cea7c993d3e999a5589a4c71669ffb1390987e43a8c4790a70275364f96cbee34b0b5a9d1b3da4322ad12e07c81c6e6430d2528f19bb1727c3f63f414885bd97505283b0bb6773712096d5feb67c43d67f2fbf17bf796ed5080aece6b968d532d985ad2553daf31ad4022aa49d7a92ada719c9f93ab4b6f0d09f127c8d47b9ab80e95a2e72257013e933113029994778c23dfa313b689e08c58979148ac541159b8eb601eee74e985a3b5b9c2dfe0ce3145794d84647136865fabf5814e8a013e236e9b740a6c18229838f3022e1aa2afe5fe48caff6e1470e4458ebcbf152462dc300b07a3d0b102a29196b0a8d444871868408fe80e1dcecd216fe8022ea70326081e516c48bd1b8a18003322738642b189013c3ed8ad64185cf1a44cb1f6265cc40450b5caea7c29b135b5145b4f3c5f14bc76f27442d5a180909ec2e144be68711737211e8d70bda6502a88a4a6558d9c857d6028b1fdfdf2d7df9d2a415b8754d194d17b29d09444d786a0478e62141c31410eda02abcd05769473e5fa75496d49aad564d139af8efee156d8089a253f4cd49814ed34fa9346701d66738938cbc5d54ba2adeb11dbe76f828dec46b82a6ff51dcf17e49771a6d88ad61996a5552809f78746562eba9d7aa9d4525d969c662628b857133d024ead8205bd3f367f3523c6ee9ff9b1784f47de41a1c196a73b178fce869b445c9a1b872a83ba946f2ca41232cdea11c53b7652dcfe615e9b3f9f0153f706eeee404e88e8736b3712e8ab9da9c9b75e419a615c3c1d1357886f77c8eaebbf4501dc1fef854fc5cc7f2f071c0a7411eb78bc14b25307cc7e4bde334ee3df0c53d6159751e82248f280434e466712dfe33981e0171e67352cdd86838eff3acd6e05592e2d1e441ddae9450a144da5c7926ee673458a59bc98c9e4d68f8b04134cc24ffdc2034c4ac3b46d6dd98ecca28dcaa7857f0c3f73a0809ae3c5dd2205555fd7cc6444024869d4f6d7dfe043917660119433c76239b17cdc8fad6c92f3756d206d67800e4e2566a73b27bb7a51dac62bc8411cdab0c5920a821e8ae6bb7779afb69f53452ad0b33c60c41a2be2c3aa94d46e97ffb5b2ebd9adc99eab85d5a3a73d4935f7ea6867d8277040c49f3ac822a4c7c7d67aba4f45765a46fccb7f79c332ac708b58911dc000c49d54fe6137be87df7f364a94fa5642338b6eeaf4152f7410ba97b0169aad82a9c4dd2d353cc3a8f57aaa90b5335f325b3e6e01", - "signature": "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505", - "committee_bits": "0x08", - "data": { - "slot": "1", - "index": "1", - "beacon_block_root": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2", - "source": { - "epoch": "1", - "root": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2" + "committee_index" : "1063251753", + "attester_index" : "1640609292", + "data" : { + "slot" : "4669978815449698508", + "index" : "4668326327938047084", + "beacon_block_root" : "0x6fdfab408c56b6105a76eff5c0435d09fc6ed7a938e7f946cf74fbbb9416428f", + "source" : { + "epoch" : "542310465", + "root" : "0x499db7404cbff78670f0209f1932346fef68d985cb55a8d27472742bdf54d379" }, - "target": { - "epoch": "1", - "root": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2" + "target" : { + "epoch" : "542695214", + "root" : "0x1f86d83f0bf91cc0d7e07410828140e0dddbb331dc20b6743f9f79e549b50b11" } - } + }, + "signature" : "0xb3a22ab9ec46aec35a9dacfb9036375ea1528041a926cb9d2d315ab964e82be5d6990e7fef2343f2dbb4c2b7dd74687f11144beaeb5758ebe349762b4dbde5e67bbc8d89a95a803c6610631d178249917cbf0d8b11bd8740f3cb767c843aa88c" } ] \ No newline at end of file From d10a47d92af7ada8757c7b16192f47b8636d6961 Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Mon, 9 Dec 2024 16:53:52 +0100 Subject: [PATCH 19/20] Fix `SingleAttestation` in `PerformanceTracker` bug (#8902) --- .../coordinator/ValidatorApiHandler.java | 13 +++-- .../DefaultPerformanceTracker.java | 2 + .../coordinator/ValidatorApiHandlerTest.java | 47 +++++++++++++++++++ 3 files changed, 59 insertions(+), 3 deletions(-) diff --git a/beacon/validator/src/main/java/tech/pegasys/teku/validator/coordinator/ValidatorApiHandler.java b/beacon/validator/src/main/java/tech/pegasys/teku/validator/coordinator/ValidatorApiHandler.java index 04648dd04fb..91515842187 100644 --- a/beacon/validator/src/main/java/tech/pegasys/teku/validator/coordinator/ValidatorApiHandler.java +++ b/beacon/validator/src/main/java/tech/pegasys/teku/validator/coordinator/ValidatorApiHandler.java @@ -586,13 +586,20 @@ public SafeFuture> sendSignedAttestations( } private SafeFuture processAttestation(final Attestation attestation) { + final ValidatableAttestation validatableAttestation = + ValidatableAttestation.fromValidator(spec, attestation); return attestationManager - .addAttestation(ValidatableAttestation.fromValidator(spec, attestation), Optional.empty()) + .addAttestation(validatableAttestation, Optional.empty()) .thenPeek( result -> { if (!result.isReject()) { - dutyMetrics.onAttestationPublished(attestation.getData().getSlot()); - performanceTracker.saveProducedAttestation(attestation); + // When saving the attestation in performance tracker, we want to make sure we save + // the converted attestation. + // The conversion happens during processing and is saved in the validatable + // attestation. + final Attestation convertedAttestation = validatableAttestation.getAttestation(); + dutyMetrics.onAttestationPublished(convertedAttestation.getData().getSlot()); + performanceTracker.saveProducedAttestation(convertedAttestation); } else { VALIDATOR_LOGGER.producedInvalidAttestation( attestation.getData().getSlot(), diff --git a/beacon/validator/src/main/java/tech/pegasys/teku/validator/coordinator/performance/DefaultPerformanceTracker.java b/beacon/validator/src/main/java/tech/pegasys/teku/validator/coordinator/performance/DefaultPerformanceTracker.java index 381229ffca5..8031f4f0a30 100644 --- a/beacon/validator/src/main/java/tech/pegasys/teku/validator/coordinator/performance/DefaultPerformanceTracker.java +++ b/beacon/validator/src/main/java/tech/pegasys/teku/validator/coordinator/performance/DefaultPerformanceTracker.java @@ -14,6 +14,7 @@ package tech.pegasys.teku.validator.coordinator.performance; import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; import com.google.common.annotations.VisibleForTesting; import it.unimi.dsi.fastutil.ints.Int2IntMap; @@ -424,6 +425,7 @@ private SafeFuture>> getAttestationsIncludedInEpoc @Override public void saveProducedAttestation(final Attestation attestation) { + checkState(!attestation.isSingleAttestation(), "Single attestation is not supported"); final UInt64 epoch = spec.computeEpochAtSlot(attestation.getData().getSlot()); final Set attestationsInEpoch = producedAttestationsByEpoch.computeIfAbsent(epoch, __ -> concurrentSet()); diff --git a/beacon/validator/src/test/java/tech/pegasys/teku/validator/coordinator/ValidatorApiHandlerTest.java b/beacon/validator/src/test/java/tech/pegasys/teku/validator/coordinator/ValidatorApiHandlerTest.java index d882571c41d..770ecbad932 100644 --- a/beacon/validator/src/test/java/tech/pegasys/teku/validator/coordinator/ValidatorApiHandlerTest.java +++ b/beacon/validator/src/test/java/tech/pegasys/teku/validator/coordinator/ValidatorApiHandlerTest.java @@ -819,6 +819,53 @@ public void sendSignedAttestations_shouldAddAttestationToAttestationManager() { .addAttestation(ValidatableAttestation.from(spec, attestation), Optional.empty()); } + @Test + void sendSignedAttestations_shouldSaveConvertedAttestationFromSingleAttestation() { + spec = TestSpecFactory.createMinimalElectra(); + dataStructureUtil = new DataStructureUtil(spec); + validatorApiHandler = + new ValidatorApiHandler( + chainDataProvider, + nodeDataProvider, + networkDataProvider, + chainDataClient, + syncStateProvider, + blockFactory, + attestationPool, + attestationManager, + attestationTopicSubscriptions, + activeValidatorTracker, + dutyMetrics, + performanceTracker, + spec, + forkChoiceTrigger, + proposersDataManager, + syncCommitteeMessagePool, + syncCommitteeContributionPool, + syncCommitteeSubscriptionManager, + blockProductionPerformanceFactory, + blockPublisher); + + final Attestation attestation = dataStructureUtil.randomSingleAttestation(); + final Attestation convertedAttestation = dataStructureUtil.randomAttestation(); + doAnswer( + invocation -> { + invocation + .getArgument(0, ValidatableAttestation.class) + .convertToAggregatedFormatFromSingleAttestation(convertedAttestation); + return completedFuture(InternalValidationResult.ACCEPT); + }) + .when(attestationManager) + .addAttestation(any(ValidatableAttestation.class), any()); + + final SafeFuture> result = + validatorApiHandler.sendSignedAttestations(List.of(attestation)); + assertThat(result).isCompletedWithValue(emptyList()); + + verify(dutyMetrics).onAttestationPublished(convertedAttestation.getData().getSlot()); + verify(performanceTracker).saveProducedAttestation(convertedAttestation); + } + @Test void sendSignedAttestations_shouldAddToDutyMetricsAndPerformanceTrackerWhenNotInvalid() { final Attestation attestation = dataStructureUtil.randomAttestation(); From a04f3ff33909018919a82e7092d62f0c831c85e9 Mon Sep 17 00:00:00 2001 From: Mehdi AOUADI Date: Mon, 9 Dec 2024 17:44:11 +0100 Subject: [PATCH 20/20] add missing electra attestation processing committee check (#8901) * add missing electra attestation processing committee check --- .../operations/OperationsTestExecutor.java | 9 --------- .../electra/block/BlockProcessorElectra.java | 16 +++++++++++++--- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/common/operations/OperationsTestExecutor.java b/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/common/operations/OperationsTestExecutor.java index 3651eae033d..df11615cc16 100644 --- a/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/common/operations/OperationsTestExecutor.java +++ b/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/common/operations/OperationsTestExecutor.java @@ -21,7 +21,6 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableMap; -import java.util.List; import java.util.Optional; import tech.pegasys.teku.ethtests.finder.TestDefinition; import tech.pegasys.teku.infrastructure.ssz.SszData; @@ -68,9 +67,6 @@ public class OperationsTestExecutor implements TestExecutor { public static final String EXPECTED_STATE_FILE = "post.ssz_snappy"; - // TODO remove https://github.com/Consensys/teku/issues/8892 - private static final List IGNORED_TEST = List.of("invalid_nonset_bits_for_one_committee"); - private enum Operation { ATTESTER_SLASHING, PROPOSER_SLASHING, @@ -148,11 +144,6 @@ public OperationsTestExecutor(final String dataFileName, final Operation operati @Override public void runTest(final TestDefinition testDefinition) throws Exception { - // TODO remove https://github.com/Consensys/teku/issues/8892 - if (IGNORED_TEST.contains(testDefinition.getTestName())) { - return; - } - final BeaconState preState = loadStateFromSsz(testDefinition, "pre.ssz_snappy"); final DefaultOperationProcessor standardProcessor = diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/electra/block/BlockProcessorElectra.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/electra/block/BlockProcessorElectra.java index 9bfe1da33b4..8693f37efc0 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/electra/block/BlockProcessorElectra.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/electra/block/BlockProcessorElectra.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.Optional; import java.util.function.Supplier; +import java.util.stream.IntStream; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.tuweni.bytes.Bytes32; @@ -725,16 +726,25 @@ private Optional checkCommittees( final BeaconState state, final UInt64 slot, final SszBitlist aggregationBits) { - int participantsCount = 0; + int committeeOffset = 0; for (final UInt64 committeeIndex : committeeIndices) { if (committeeIndex.isGreaterThanOrEqualTo(committeeCountPerSlot)) { return Optional.of(AttestationInvalidReason.COMMITTEE_INDEX_TOO_HIGH); } final IntList committee = beaconStateAccessorsElectra.getBeaconCommittee(state, slot, committeeIndex); - participantsCount += committee.size(); + final int currentCommitteeOffset = committeeOffset; + final boolean committeeHasAtLeastOneAttester = + IntStream.range(0, committee.size()) + .anyMatch( + committeeParticipantIndex -> + aggregationBits.isSet(currentCommitteeOffset + committeeParticipantIndex)); + if (!committeeHasAtLeastOneAttester) { + return Optional.of(AttestationInvalidReason.PARTICIPANTS_COUNT_MISMATCH); + } + committeeOffset += committee.size(); } - if (participantsCount != aggregationBits.size()) { + if (committeeOffset != aggregationBits.size()) { return Optional.of(AttestationInvalidReason.PARTICIPANTS_COUNT_MISMATCH); } return Optional.empty();