From ac333a758d5d70b717ea836df360139c73870b6e Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Fri, 29 Sep 2023 19:24:51 +0200 Subject: [PATCH 1/4] implement blob_sidecar event --- .../beaconrestapi/BeaconRestApiTypes.java | 2 +- .../JsonTypeDefinitionBeaconRestApi.java | 1 + .../handlers/v1/events/BlobSidecarEvent.java | 104 ++++++++++++++++++ .../v1/events/EventSubscriptionManager.java | 11 ++ .../handlers/v1/events/GetEvents.java | 5 + .../events/EventSubscriptionManagerTest.java | 28 ++++- .../tech/pegasys/teku/api/DataProvider.java | 8 ++ .../pegasys/teku/api/NodeDataProvider.java | 9 ++ .../teku/api/NodeDataProviderTest.java | 4 + .../teku/api/response/v1/EventType.java | 3 +- .../ethereum/json/types/EthereumTypes.java | 21 ++++ .../versions/deneb/types/VersionedHash.java | 4 + .../blobs/BlobSidecarPool.java | 9 ++ .../util/BlobSidecarPoolImpl.java | 22 +++- .../util/BlobSidecarPoolImplTest.java | 9 ++ .../infrastructure/json/types/CoreTypes.java | 10 -- .../beaconchain/BeaconChainController.java | 1 + 17 files changed, 237 insertions(+), 14 deletions(-) create mode 100644 data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/BlobSidecarEvent.java diff --git a/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/BeaconRestApiTypes.java b/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/BeaconRestApiTypes.java index e5dce46d26a..f4e3a87a1e3 100644 --- a/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/BeaconRestApiTypes.java +++ b/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/BeaconRestApiTypes.java @@ -186,7 +186,7 @@ public class BeaconRestApiTypes { CoreTypes.string( "Event types to subscribe to." + " Available values include: [`head`, `finalized_checkpoint`, `chain_reorg`, `block`, " - + "`attestation`, `voluntary_exit`, `contribution_and_proof`]\n\n", + + "`attestation`, `voluntary_exit`, `contribution_and_proof`, `blob_sidecar`]\n\n", "head")); public static final SerializableTypeDefinition ROOT_TYPE = diff --git a/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/JsonTypeDefinitionBeaconRestApi.java b/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/JsonTypeDefinitionBeaconRestApi.java index da2b7b1f95c..e6367a2eff5 100644 --- a/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/JsonTypeDefinitionBeaconRestApi.java +++ b/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/JsonTypeDefinitionBeaconRestApi.java @@ -242,6 +242,7 @@ private static RestApi create( // Event Handler .endpoint( new GetEvents( + spec, dataProvider, eventChannels, asyncRunner, diff --git a/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/BlobSidecarEvent.java b/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/BlobSidecarEvent.java new file mode 100644 index 00000000000..d976b759956 --- /dev/null +++ b/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/BlobSidecarEvent.java @@ -0,0 +1,104 @@ +/* + * Copyright Consensys Software Inc., 2022 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.beaconrestapi.handlers.v1.events; + +import static tech.pegasys.teku.ethereum.json.types.EthereumTypes.KZG_COMMITMENT_TYPE; +import static tech.pegasys.teku.ethereum.json.types.EthereumTypes.VERSIONED_HASH_TYPE; +import static tech.pegasys.teku.infrastructure.json.types.CoreTypes.BYTES32_TYPE; +import static tech.pegasys.teku.infrastructure.json.types.CoreTypes.UINT64_TYPE; + +import org.apache.tuweni.bytes.Bytes32; +import tech.pegasys.teku.beaconrestapi.handlers.v1.events.BlobSidecarEvent.BlobSidecarData; +import tech.pegasys.teku.infrastructure.json.types.SerializableTypeDefinition; +import tech.pegasys.teku.infrastructure.unsigned.UInt64; +import tech.pegasys.teku.kzg.KZGCommitment; +import tech.pegasys.teku.spec.Spec; +import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; +import tech.pegasys.teku.spec.logic.versions.deneb.types.VersionedHash; + +public class BlobSidecarEvent extends Event { + + public static final SerializableTypeDefinition BLOB_SIDECAR_EVENT_TYPE = + SerializableTypeDefinition.object(BlobSidecarData.class) + .name("BlobSidecarEvent") + .withField("block_root", BYTES32_TYPE, BlobSidecarData::getBlockRoot) + .withField("index", UINT64_TYPE, BlobSidecarData::getIndex) + .withField("slot", UINT64_TYPE, BlobSidecarData::getSlot) + .withField("kzg_commitment", KZG_COMMITMENT_TYPE, BlobSidecarData::getKzgCommitment) + .withField("versioned_hash", VERSIONED_HASH_TYPE, BlobSidecarData::getVersionedHash) + .build(); + + private BlobSidecarEvent( + final Bytes32 blockRoot, + final UInt64 index, + final UInt64 slot, + final KZGCommitment kzgCommitment, + final VersionedHash versionedHash) { + super( + BLOB_SIDECAR_EVENT_TYPE, + new BlobSidecarData(blockRoot, index, slot, kzgCommitment, versionedHash)); + } + + public static BlobSidecarEvent create(final Spec spec, final BlobSidecar blobSidecar) { + return new BlobSidecarEvent( + blobSidecar.getBlockRoot(), + blobSidecar.getIndex(), + blobSidecar.getSlot(), + blobSidecar.getKZGCommitment(), + spec.atSlot(blobSidecar.getSlot()) + .miscHelpers() + .kzgCommitmentToVersionedHash(blobSidecar.getKZGCommitment())); + } + + public static class BlobSidecarData { + private final Bytes32 blockRoot; + private final UInt64 index; + private final UInt64 slot; + private final KZGCommitment kzgCommitment; + private final VersionedHash versionedHash; + + BlobSidecarData( + final Bytes32 blockRoot, + final UInt64 index, + final UInt64 slot, + final KZGCommitment kzgCommitment, + final VersionedHash versionedHash) { + this.blockRoot = blockRoot; + this.index = index; + this.slot = slot; + this.kzgCommitment = kzgCommitment; + this.versionedHash = versionedHash; + } + + public Bytes32 getBlockRoot() { + return blockRoot; + } + + public UInt64 getIndex() { + return index; + } + + public UInt64 getSlot() { + return slot; + } + + public KZGCommitment getKzgCommitment() { + return kzgCommitment; + } + + public VersionedHash getVersionedHash() { + return versionedHash; + } + } +} diff --git a/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java b/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java index 1abe225712b..d83d9c14f6a 100644 --- a/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java +++ b/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java @@ -38,7 +38,9 @@ import tech.pegasys.teku.infrastructure.restapi.endpoints.ListQueryParameterUtils; import tech.pegasys.teku.infrastructure.time.TimeProvider; import tech.pegasys.teku.infrastructure.unsigned.UInt64; +import tech.pegasys.teku.spec.Spec; import tech.pegasys.teku.spec.datastructures.attestation.ValidatableAttestation; +import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; import tech.pegasys.teku.spec.datastructures.operations.SignedBlsToExecutionChange; import tech.pegasys.teku.spec.datastructures.operations.SignedVoluntaryExit; @@ -52,6 +54,7 @@ public class EventSubscriptionManager implements ChainHeadChannel, FinalizedCheckpointChannel { private static final Logger LOG = LogManager.getLogger(); + private Spec spec; private final ConfigProvider configProvider; private final ChainDataProvider provider; private final AsyncRunner asyncRunner; @@ -61,6 +64,7 @@ public class EventSubscriptionManager implements ChainHeadChannel, FinalizedChec private final Collection eventSubscribers; public EventSubscriptionManager( + final Spec spec, final NodeDataProvider nodeDataProvider, final ChainDataProvider chainDataProvider, final SyncDataProvider syncDataProvider, @@ -69,6 +73,7 @@ public EventSubscriptionManager( final EventChannels eventChannels, final TimeProvider timeProvider, final int maxPendingEvents) { + this.spec = spec; this.provider = chainDataProvider; this.asyncRunner = asyncRunner; this.timeProvider = timeProvider; @@ -79,6 +84,7 @@ public EventSubscriptionManager( eventChannels.subscribe(FinalizedCheckpointChannel.class, this); syncDataProvider.subscribeToSyncStateChanges(this::onSyncStateChange); nodeDataProvider.subscribeToReceivedBlocks(this::onNewBlock); + nodeDataProvider.subscribeToReceivedBlobSidecar(this::onNewBlobSidecar); nodeDataProvider.subscribeToValidAttestations(this::onNewAttestation); nodeDataProvider.subscribeToNewVoluntaryExits(this::onNewVoluntaryExit); nodeDataProvider.subscribeToSyncCommitteeContributions(this::onSyncCommitteeContribution); @@ -182,6 +188,11 @@ protected void onNewBlock(final SignedBeaconBlock block, final boolean execution notifySubscribersOfEvent(EventType.block, blockEvent); } + protected void onNewBlobSidecar(final BlobSidecar blobSidecar) { + final BlobSidecarEvent blobSidecarEvent = BlobSidecarEvent.create(spec, blobSidecar); + notifySubscribersOfEvent(EventType.blob_sidecar, blobSidecarEvent); + } + @Override public void onNewFinalizedCheckpoint( final Checkpoint checkpoint, final boolean fromOptimisticBlock) { diff --git a/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/GetEvents.java b/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/GetEvents.java index 8d77e5ed80e..eab07771370 100644 --- a/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/GetEvents.java +++ b/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/GetEvents.java @@ -31,18 +31,21 @@ import tech.pegasys.teku.infrastructure.restapi.endpoints.RestApiRequest; import tech.pegasys.teku.infrastructure.restapi.openapi.response.EventStreamResponseContentTypeDefinition; import tech.pegasys.teku.infrastructure.time.TimeProvider; +import tech.pegasys.teku.spec.Spec; public class GetEvents extends RestApiEndpoint { public static final String ROUTE = "/eth/v1/events"; private final EventSubscriptionManager eventSubscriptionManager; public GetEvents( + final Spec spec, final DataProvider dataProvider, final EventChannels eventChannels, final AsyncRunner asyncRunner, final TimeProvider timeProvider, final int maxPendingEvents) { this( + spec, dataProvider.getNodeDataProvider(), dataProvider.getChainDataProvider(), dataProvider.getSyncDataProvider(), @@ -54,6 +57,7 @@ public GetEvents( } GetEvents( + final Spec spec, final NodeDataProvider nodeDataProvider, final ChainDataProvider chainDataProvider, final SyncDataProvider syncDataProvider, @@ -78,6 +82,7 @@ public GetEvents( .build()); eventSubscriptionManager = new EventSubscriptionManager( + spec, nodeDataProvider, chainDataProvider, syncDataProvider, diff --git a/data/beaconrestapi/src/test/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManagerTest.java b/data/beaconrestapi/src/test/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManagerTest.java index 99bcd6fbdfb..1f557c760f8 100644 --- a/data/beaconrestapi/src/test/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManagerTest.java +++ b/data/beaconrestapi/src/test/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManagerTest.java @@ -44,6 +44,7 @@ import tech.pegasys.teku.spec.TestSpecFactory; import tech.pegasys.teku.spec.config.SpecConfig; import tech.pegasys.teku.spec.datastructures.attestation.ValidatableAttestation; +import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; import tech.pegasys.teku.spec.datastructures.operations.Attestation; import tech.pegasys.teku.spec.datastructures.operations.SignedBlsToExecutionChange; import tech.pegasys.teku.spec.datastructures.operations.SignedVoluntaryExit; @@ -54,7 +55,7 @@ import tech.pegasys.teku.storage.api.ReorgContext; public class EventSubscriptionManagerTest { - private final Spec spec = TestSpecFactory.createMinimalCapella(); + private final Spec spec = TestSpecFactory.createMinimalDeneb(); private final SpecConfig specConfig = spec.getGenesisSpecConfig(); private final DataStructureUtil data = new DataStructureUtil(spec); protected final NodeDataProvider nodeDataProvider = mock(NodeDataProvider.class); @@ -94,6 +95,7 @@ public class EventSubscriptionManagerTest { private final SyncState sampleSyncState = SyncState.IN_SYNC; private final SignedBeaconBlock sampleBlock = SignedBeaconBlock.create(data.randomSignedBeaconBlock(0)); + private final BlobSidecar sampleBlobSidecar = data.randomBlobSidecar(); private final Attestation sampleAttestation = data.randomAttestation(0); private final SignedVoluntaryExit sampleVoluntaryExit = data.randomSignedVoluntaryExit(); private final SignedBlsToExecutionChange sampleBlsToExecutionChange = @@ -116,6 +118,7 @@ void setup() throws IOException { when(res.getOutputStream()).thenReturn(outputStream); manager = new EventSubscriptionManager( + spec, nodeDataProvider, chainDataProvider, syncDataProvider, @@ -205,6 +208,15 @@ void shouldPropagateBlock() throws IOException { checkEvent("block", new BlockEvent(sampleBlock.asInternalSignedBeaconBlock(spec), false)); } + @Test + void shouldPropagateBlobSidecar() throws IOException { + when(req.getQueryString()).thenReturn("&topics=blob_sidecar"); + manager.registerClient(client1); + + triggerBlobSidecarEvent(); + checkEvent("blob_sidecar", BlobSidecarEvent.create(spec, sampleBlobSidecar)); + } + @Test void shouldPropagateAttestation() throws IOException { when(req.getQueryString()).thenReturn("&topics=attestation"); @@ -258,6 +270,15 @@ void shouldNotGetBlockIfNotSubscribed() { assertThat(outputStream.countEvents()).isEqualTo(0); } + @Test + void shouldNotGetBlobSidecarIfNotSubscribed() { + when(req.getQueryString()).thenReturn("&topics=head"); + manager.registerClient(client1); + + triggerBlobSidecarEvent(); + assertThat(outputStream.countEvents()).isEqualTo(0); + } + @Test void shouldNotGetAttestationIfNotSubscribed() { when(req.getQueryString()).thenReturn("&topics=head"); @@ -315,6 +336,11 @@ private void triggerBlockEvent() { asyncRunner.executeQueuedActions(); } + private void triggerBlobSidecarEvent() { + manager.onNewBlobSidecar(sampleBlobSidecar); + asyncRunner.executeQueuedActions(); + } + private void triggerSyncStateEvent() { manager.onSyncStateChange(sampleSyncState); asyncRunner.executeQueuedActions(); diff --git a/data/provider/src/main/java/tech/pegasys/teku/api/DataProvider.java b/data/provider/src/main/java/tech/pegasys/teku/api/DataProvider.java index 29870ae0a16..10e0ba6ec37 100644 --- a/data/provider/src/main/java/tech/pegasys/teku/api/DataProvider.java +++ b/data/provider/src/main/java/tech/pegasys/teku/api/DataProvider.java @@ -27,6 +27,7 @@ import tech.pegasys.teku.statetransition.OperationPool; import tech.pegasys.teku.statetransition.attestation.AggregatingAttestationPool; import tech.pegasys.teku.statetransition.attestation.AttestationManager; +import tech.pegasys.teku.statetransition.blobs.BlobSidecarPool; import tech.pegasys.teku.statetransition.block.BlockManager; import tech.pegasys.teku.statetransition.forkchoice.ProposersDataManager; import tech.pegasys.teku.statetransition.synccommittee.SyncCommitteeContributionPool; @@ -97,6 +98,7 @@ public static class Builder { private ValidatorApiChannel validatorApiChannel; private AggregatingAttestationPool attestationPool; private BlockManager blockManager; + private BlobSidecarPool blobSidecarPool; private AttestationManager attestationManager; private ActiveValidatorChannel activeValidatorChannel; private OperationPool attesterSlashingPool; @@ -145,6 +147,11 @@ public Builder blockManager(final BlockManager blockManager) { return this; } + public Builder blobSidecarPool(final BlobSidecarPool blobSidecarPool) { + this.blobSidecarPool = blobSidecarPool; + return this; + } + public Builder attestationManager(final AttestationManager attestationManager) { this.attestationManager = attestationManager; return this; @@ -216,6 +223,7 @@ public DataProvider build() { blsToExecutionChangePool, syncCommitteeContributionPool, blockManager, + blobSidecarPool, attestationManager, isLivenessTrackingEnabled, activeValidatorChannel, diff --git a/data/provider/src/main/java/tech/pegasys/teku/api/NodeDataProvider.java b/data/provider/src/main/java/tech/pegasys/teku/api/NodeDataProvider.java index 7818669b93d..7ba0f1ff8ad 100644 --- a/data/provider/src/main/java/tech/pegasys/teku/api/NodeDataProvider.java +++ b/data/provider/src/main/java/tech/pegasys/teku/api/NodeDataProvider.java @@ -38,6 +38,8 @@ import tech.pegasys.teku.statetransition.OperationPool; import tech.pegasys.teku.statetransition.attestation.AggregatingAttestationPool; import tech.pegasys.teku.statetransition.attestation.AttestationManager; +import tech.pegasys.teku.statetransition.blobs.BlobSidecarPool; +import tech.pegasys.teku.statetransition.blobs.BlobSidecarPool.NewBlobSidecarSubscriber; import tech.pegasys.teku.statetransition.block.BlockManager; import tech.pegasys.teku.statetransition.forkchoice.PreparedProposerInfo; import tech.pegasys.teku.statetransition.forkchoice.ProposersDataManager; @@ -56,6 +58,7 @@ public class NodeDataProvider { private final OperationPool blsToExecutionChangePool; private final SyncCommitteeContributionPool syncCommitteeContributionPool; private final BlockManager blockManager; + private final BlobSidecarPool blobSidecarPool; private final AttestationManager attestationManager; private final ActiveValidatorChannel activeValidatorChannel; private final boolean isLivenessTrackingEnabled; @@ -70,6 +73,7 @@ public NodeDataProvider( final OperationPool blsToExecutionChangePool, final SyncCommitteeContributionPool syncCommitteeContributionPool, final BlockManager blockManager, + final BlobSidecarPool blobSidecarPool, final AttestationManager attestationManager, final boolean isLivenessTrackingEnabled, final ActiveValidatorChannel activeValidatorChannel, @@ -82,6 +86,7 @@ public NodeDataProvider( this.blsToExecutionChangePool = blsToExecutionChangePool; this.syncCommitteeContributionPool = syncCommitteeContributionPool; this.blockManager = blockManager; + this.blobSidecarPool = blobSidecarPool; this.attestationManager = attestationManager; this.activeValidatorChannel = activeValidatorChannel; this.isLivenessTrackingEnabled = isLivenessTrackingEnabled; @@ -172,6 +177,10 @@ public void subscribeToReceivedBlocks(ImportedBlockListener listener) { blockManager.subscribeToReceivedBlocks(listener); } + public void subscribeToReceivedBlobSidecar(NewBlobSidecarSubscriber listener) { + blobSidecarPool.subscribeNewBlobSidecar(listener); + } + public void subscribeToValidAttestations(ProcessedAttestationListener listener) { attestationManager.subscribeToAllValidAttestations(listener); } diff --git a/data/provider/src/test/java/tech/pegasys/teku/api/NodeDataProviderTest.java b/data/provider/src/test/java/tech/pegasys/teku/api/NodeDataProviderTest.java index 914a94050b5..dd363a8c4cd 100644 --- a/data/provider/src/test/java/tech/pegasys/teku/api/NodeDataProviderTest.java +++ b/data/provider/src/test/java/tech/pegasys/teku/api/NodeDataProviderTest.java @@ -36,6 +36,7 @@ import tech.pegasys.teku.statetransition.OperationPool; import tech.pegasys.teku.statetransition.attestation.AggregatingAttestationPool; import tech.pegasys.teku.statetransition.attestation.AttestationManager; +import tech.pegasys.teku.statetransition.blobs.BlobSidecarPool; import tech.pegasys.teku.statetransition.block.BlockManager; import tech.pegasys.teku.statetransition.forkchoice.ProposersDataManager; import tech.pegasys.teku.statetransition.synccommittee.SyncCommitteeContributionPool; @@ -50,6 +51,7 @@ public class NodeDataProviderTest { private final AggregatingAttestationPool attestationPool = mock(AggregatingAttestationPool.class); private final BlockManager blockManager = mock(BlockManager.class); + private final BlobSidecarPool blobSidecarPool = mock(BlobSidecarPool.class); private final AttestationManager attestationManager = mock(AttestationManager.class); private final ActiveValidatorChannel validatorChannel = mock(ActiveValidatorChannel.class); private final ProposersDataManager proposersDataManager = mock(ProposersDataManager.class); @@ -78,6 +80,7 @@ public void setup() { blsToExecutionChangePool, syncCommitteeContributionPool, blockManager, + blobSidecarPool, attestationManager, false, validatorChannel, @@ -125,6 +128,7 @@ public void shouldReturnErrorWhenPostingBlsToExecutionChangesAndAcceptFlagIsFals blsToExecutionChangePool, syncCommitteeContributionPool, blockManager, + blobSidecarPool, attestationManager, false, validatorChannel, diff --git a/data/serializer/src/main/java/tech/pegasys/teku/api/response/v1/EventType.java b/data/serializer/src/main/java/tech/pegasys/teku/api/response/v1/EventType.java index bf53bb22d1d..767401fa28d 100644 --- a/data/serializer/src/main/java/tech/pegasys/teku/api/response/v1/EventType.java +++ b/data/serializer/src/main/java/tech/pegasys/teku/api/response/v1/EventType.java @@ -25,7 +25,8 @@ public enum EventType { chain_reorg, sync_state, contribution_and_proof, - bls_to_execution_change; + bls_to_execution_change, + blob_sidecar; public static List getTopics(List topics) { return topics.stream().map(EventType::valueOf).toList(); diff --git a/ethereum/json-types/src/main/java/tech/pegasys/teku/ethereum/json/types/EthereumTypes.java b/ethereum/json-types/src/main/java/tech/pegasys/teku/ethereum/json/types/EthereumTypes.java index 21a3db707e1..4e9c2e66100 100644 --- a/ethereum/json-types/src/main/java/tech/pegasys/teku/ethereum/json/types/EthereumTypes.java +++ b/ethereum/json-types/src/main/java/tech/pegasys/teku/ethereum/json/types/EthereumTypes.java @@ -29,8 +29,10 @@ import tech.pegasys.teku.infrastructure.restapi.openapi.response.OctetStreamResponseContentTypeDefinition; import tech.pegasys.teku.infrastructure.restapi.openapi.response.ResponseContentTypeDefinition; import tech.pegasys.teku.infrastructure.ssz.SszData; +import tech.pegasys.teku.kzg.KZGCommitment; import tech.pegasys.teku.spec.SpecMilestone; import tech.pegasys.teku.spec.datastructures.metadata.ObjectAndMetaData; +import tech.pegasys.teku.spec.logic.versions.deneb.types.VersionedHash; public class EthereumTypes { @@ -68,6 +70,25 @@ public class EthereumTypes { .format("string") .build(); + public static final StringValueTypeDefinition KZG_COMMITMENT_TYPE = + DeserializableTypeDefinition.string(KZGCommitment.class) + .formatter(KZGCommitment::toHexString) + .parser(KZGCommitment::fromHexString) + .example( + "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505") + .description("KZG Commitments") + .format("byte") + .build(); + + public static final StringValueTypeDefinition VERSIONED_HASH_TYPE = + DeserializableTypeDefinition.string(VersionedHash.class) + .formatter(VersionedHash::toHexString) + .parser(VersionedHash::fromHexString) + .example("0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2") + .description("Versioned Hash") + .format("byte") + .build(); + public static final DeserializableTypeDefinition MILESTONE_TYPE = DeserializableTypeDefinition.enumOf( SpecMilestone.class, milestone -> milestone.name().toLowerCase(Locale.ROOT), Set.of()); diff --git a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/types/VersionedHash.java b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/types/VersionedHash.java index 1e06b199b38..46902f579fe 100644 --- a/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/types/VersionedHash.java +++ b/ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/versions/deneb/types/VersionedHash.java @@ -41,6 +41,10 @@ public static VersionedHash create(final Bytes version, final Bytes32 hash) { return new VersionedHash(version, hash.slice(1)); } + public static VersionedHash fromHexString(final String hexVersionedHash) { + return new VersionedHash(Bytes32.fromHexString(hexVersionedHash)); + } + public Bytes getVersion() { return version; } diff --git a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarPool.java b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarPool.java index 3abe1d88622..8448a96812c 100644 --- a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarPool.java +++ b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/blobs/BlobSidecarPool.java @@ -87,6 +87,9 @@ public void subscribeRequiredBlockRoot( @Override public void subscribeRequiredBlockRootDropped( final RequiredBlockRootDroppedSubscriber requiredBlockRootDroppedSubscriber) {} + + @Override + public void subscribeNewBlobSidecar(NewBlobSidecarSubscriber newBlobSidecarSubscriber) {} }; void onNewBlobSidecar(BlobSidecar blobSidecar); @@ -123,6 +126,8 @@ void subscribeRequiredBlobSidecarDropped( void subscribeRequiredBlockRootDropped( RequiredBlockRootDroppedSubscriber requiredBlockRootDroppedSubscriber); + void subscribeNewBlobSidecar(NewBlobSidecarSubscriber newBlobSidecarSubscriber); + interface RequiredBlobSidecarSubscriber { void onRequiredBlobSidecar(BlobIdentifier blobIdentifier); } @@ -138,4 +143,8 @@ interface RequiredBlockRootSubscriber { interface RequiredBlockRootDroppedSubscriber { void onRequiredBlockRootDropped(Bytes32 blockRoot); } + + interface NewBlobSidecarSubscriber { + void onNewBlobSidecar(BlobSidecar blobSidecar); + } } diff --git a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/util/BlobSidecarPoolImpl.java b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/util/BlobSidecarPoolImpl.java index 5df7d85097e..f6f68a09de9 100644 --- a/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/util/BlobSidecarPoolImpl.java +++ b/ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/util/BlobSidecarPoolImpl.java @@ -77,6 +77,9 @@ public class BlobSidecarPoolImpl extends AbstractIgnoringFutureHistoricalSlot private final Subscribers requiredBlobSidecarDroppedSubscribers = Subscribers.create(true); + private final Subscribers newBlobSidecarSubscribers = + Subscribers.create(true); + private int totalBlobSidecars; BlobSidecarPoolImpl( @@ -151,6 +154,7 @@ public synchronized void onNewBlobSidecar(final BlobSidecar blobSidecar) { if (blobSidecarsTracker.add(blobSidecar)) { sizeGauge.set(++totalBlobSidecars, GAUGE_BLOB_SIDECARS_LABEL); + newBlobSidecarSubscribers.deliver(NewBlobSidecarSubscriber::onNewBlobSidecar, blobSidecar); } if (orderedBlobSidecarsTrackers.add(slotAndBlockRoot)) { @@ -192,7 +196,18 @@ public synchronized void onCompletedBlockAndBlobSidecars( blobSidecarsTracker.setBlock(block); long addedBlobs = - blobSidecars.stream().map(blobSidecarsTracker::add).filter(Boolean::booleanValue).count(); + blobSidecars.stream() + .map( + blobSidecar -> { + final boolean isNew = blobSidecarsTracker.add(blobSidecar); + if (isNew) { + newBlobSidecarSubscribers.deliver( + NewBlobSidecarSubscriber::onNewBlobSidecar, blobSidecar); + } + return isNew; + }) + .filter(Boolean::booleanValue) + .count(); totalBlobSidecars += (int) addedBlobs; sizeGauge.set(totalBlobSidecars, GAUGE_BLOB_SIDECARS_LABEL); @@ -287,6 +302,11 @@ public void subscribeRequiredBlockRootDropped( requiredBlockRootDroppedSubscribers.subscribe(requiredBlockRootDroppedSubscriber); } + @Override + public void subscribeNewBlobSidecar(final NewBlobSidecarSubscriber newBlobSidecarSubscriber) { + newBlobSidecarSubscribers.subscribe(newBlobSidecarSubscriber); + } + public synchronized int getTotalBlobSidecars() { return totalBlobSidecars; } diff --git a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/util/BlobSidecarPoolImplTest.java b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/util/BlobSidecarPoolImplTest.java index 6332e9a2776..aa4598f7cf5 100644 --- a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/util/BlobSidecarPoolImplTest.java +++ b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/util/BlobSidecarPoolImplTest.java @@ -81,6 +81,7 @@ public class BlobSidecarPoolImplTest { private final List requiredBlockRootDroppedEvents = new ArrayList<>(); private final List requiredBlobSidecarEvents = new ArrayList<>(); private final List requiredBlobSidecarDroppedEvents = new ArrayList<>(); + private final List newBlobSidecarEvents = new ArrayList<>(); private Optional> mockedTrackersFactory = Optional.empty(); @@ -92,6 +93,7 @@ public void setup() { blobSidecarPool.subscribeRequiredBlockRootDropped(requiredBlockRootDroppedEvents::add); blobSidecarPool.subscribeRequiredBlobSidecar(requiredBlobSidecarEvents::add); blobSidecarPool.subscribeRequiredBlobSidecarDropped(requiredBlobSidecarDroppedEvents::add); + blobSidecarPool.subscribeNewBlobSidecar(newBlobSidecarEvents::add); setSlot(currentSlot); } @@ -134,6 +136,7 @@ public void onNewBlobSidecar_addTrackerWithBlobSidecar() { assertThat(requiredBlockRootDroppedEvents).isEmpty(); assertThat(requiredBlobSidecarEvents).isEmpty(); assertThat(requiredBlobSidecarDroppedEvents).isEmpty(); + assertThat(newBlobSidecarEvents).containsExactly(blobSidecar); assertBlobSidecarsCount(1); assertBlobSidecarsTrackersCount(1); @@ -176,6 +179,7 @@ public void onNewBlobSidecar_shouldIgnoreBlobsForAlreadyImportedBlocks() { assertThat(requiredBlockRootDroppedEvents).isEmpty(); assertThat(requiredBlobSidecarEvents).isEmpty(); assertThat(requiredBlobSidecarDroppedEvents).isEmpty(); + assertThat(newBlobSidecarEvents).isEmpty(); assertBlobSidecarsCount(0); assertBlobSidecarsTrackersCount(0); @@ -200,6 +204,7 @@ public void onNewBlobSidecarOnNewBlock_addTrackerWithBothBlockAndBlobSidecar() { assertThat(requiredBlockRootEvents).isEmpty(); assertThat(requiredBlockRootDroppedEvents).isEmpty(); assertThat(requiredBlobSidecarEvents).isEmpty(); + assertThat(newBlobSidecarEvents).containsExactly(blobSidecar); assertBlobSidecarsCount(1); assertBlobSidecarsTrackersCount(1); @@ -245,6 +250,7 @@ public void twoOnNewBlobSidecar_addTrackerWithBothBlobSidecars() { assertThat(requiredBlockRootDroppedEvents).isEmpty(); assertThat(requiredBlobSidecarEvents).isEmpty(); assertThat(requiredBlobSidecarDroppedEvents).isEmpty(); + assertThat(newBlobSidecarEvents).containsExactly(blobSidecar0, blobSidecar1); assertBlobSidecarsCount(2); assertBlobSidecarsTrackersCount(1); @@ -267,6 +273,7 @@ public void twoOnNewBlock_addTrackerWithBothBlobSidecars() { assertThat(requiredBlockRootDroppedEvents).isEmpty(); assertThat(requiredBlobSidecarEvents).isEmpty(); assertThat(requiredBlobSidecarDroppedEvents).isEmpty(); + assertThat(newBlobSidecarEvents).isEmpty(); assertBlobSidecarsCount(0); assertBlobSidecarsTrackersCount(2); @@ -288,6 +295,7 @@ public void onCompletedBlockAndBlobSidecars_shouldCreateTrackerIgnoringHistorica assertThat(requiredBlockRootDroppedEvents).isEmpty(); assertThat(requiredBlobSidecarEvents).isEmpty(); assertThat(requiredBlobSidecarDroppedEvents).isEmpty(); + assertThat(newBlobSidecarEvents).containsExactlyElementsOf(blobSidecars); final BlockBlobSidecarsTracker blockBlobSidecarsTracker = blobSidecarPool.getBlobSidecarsTracker(block.getSlotAndBlockRoot()); @@ -317,6 +325,7 @@ public void onCompletedBlockAndBlobSidecars_shouldNotTriggerFetch() { assertThat(requiredBlockRootDroppedEvents).isEmpty(); assertThat(requiredBlobSidecarEvents).isEmpty(); assertThat(requiredBlobSidecarDroppedEvents).isEmpty(); + assertThat(newBlobSidecarEvents).containsExactlyElementsOf(blobSidecars); final BlockBlobSidecarsTracker blockBlobSidecarsTracker = blobSidecarPool.getBlobSidecarsTracker(block.getSlotAndBlockRoot()); diff --git a/infrastructure/json/src/main/java/tech/pegasys/teku/infrastructure/json/types/CoreTypes.java b/infrastructure/json/src/main/java/tech/pegasys/teku/infrastructure/json/types/CoreTypes.java index e373e0d36d6..179ac721a9f 100644 --- a/infrastructure/json/src/main/java/tech/pegasys/teku/infrastructure/json/types/CoreTypes.java +++ b/infrastructure/json/src/main/java/tech/pegasys/teku/infrastructure/json/types/CoreTypes.java @@ -17,7 +17,6 @@ import java.util.function.Function; import org.apache.tuweni.bytes.Bytes32; import org.apache.tuweni.units.bigints.UInt256; -import tech.pegasys.teku.infrastructure.bytes.Bytes20; import tech.pegasys.teku.infrastructure.bytes.Bytes4; import tech.pegasys.teku.infrastructure.http.HttpErrorResponse; import tech.pegasys.teku.infrastructure.json.types.StringBasedPrimitiveTypeDefinition.StringTypeBuilder; @@ -39,15 +38,6 @@ public class CoreTypes { .format("byte") .build(); - public static final StringValueTypeDefinition BYTES20_TYPE = - DeserializableTypeDefinition.string(Bytes20.class) - .formatter(Bytes20::toHexString) - .parser(Bytes20::fromHexString) - .example("0xcf8e0d4e9587369b2301d0790347320302cc0943") - .description("Bytes20 hexadecimal") - .format("byte") - .build(); - public static final DeserializableTypeDefinition BYTES4_TYPE = DeserializableTypeDefinition.string(Bytes4.class) .formatter(Bytes4::toHexString) diff --git a/services/beaconchain/src/main/java/tech/pegasys/teku/services/beaconchain/BeaconChainController.java b/services/beaconchain/src/main/java/tech/pegasys/teku/services/beaconchain/BeaconChainController.java index 532d8cb4695..131984bf4b7 100644 --- a/services/beaconchain/src/main/java/tech/pegasys/teku/services/beaconchain/BeaconChainController.java +++ b/services/beaconchain/src/main/java/tech/pegasys/teku/services/beaconchain/BeaconChainController.java @@ -622,6 +622,7 @@ protected void initDataProvider() { eventChannels.getPublisher(ValidatorApiChannel.class, beaconAsyncRunner)) .attestationPool(attestationPool) .blockManager(blockManager) + .blobSidecarPool(blobSidecarPool) .attestationManager(attestationManager) .isLivenessTrackingEnabled(getLivenessTrackingEnabled(beaconConfig)) .acceptBlsToExecutionMessages( From 57aaa970ae40d6d5ba2b8096b4bdfd1d82d75ac4 Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Fri, 29 Sep 2023 19:39:17 +0200 Subject: [PATCH 2/4] spotlessly --- .../test/java/tech/pegasys/teku/api/NodeDataProviderTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/provider/src/test/java/tech/pegasys/teku/api/NodeDataProviderTest.java b/data/provider/src/test/java/tech/pegasys/teku/api/NodeDataProviderTest.java index dd363a8c4cd..b234c11dced 100644 --- a/data/provider/src/test/java/tech/pegasys/teku/api/NodeDataProviderTest.java +++ b/data/provider/src/test/java/tech/pegasys/teku/api/NodeDataProviderTest.java @@ -80,7 +80,7 @@ public void setup() { blsToExecutionChangePool, syncCommitteeContributionPool, blockManager, - blobSidecarPool, + blobSidecarPool, attestationManager, false, validatorChannel, @@ -128,7 +128,7 @@ public void shouldReturnErrorWhenPostingBlsToExecutionChangesAndAcceptFlagIsFals blsToExecutionChangePool, syncCommitteeContributionPool, blockManager, - blobSidecarPool, + blobSidecarPool, attestationManager, false, validatorChannel, From 8267fb65a2ffdaff28d39ea727c8965240a278ac Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Fri, 29 Sep 2023 20:01:02 +0200 Subject: [PATCH 3/4] fix integration tests --- .../beaconrestapi/AbstractDataBackedRestAPIIntegrationTest.java | 2 ++ .../pegasys/teku/beaconrestapi/beacon/paths/_eth_v1_events.json | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/data/beaconrestapi/src/integration-test/java/tech/pegasys/teku/beaconrestapi/AbstractDataBackedRestAPIIntegrationTest.java b/data/beaconrestapi/src/integration-test/java/tech/pegasys/teku/beaconrestapi/AbstractDataBackedRestAPIIntegrationTest.java index e4dc8fd496f..89f942f2f28 100644 --- a/data/beaconrestapi/src/integration-test/java/tech/pegasys/teku/beaconrestapi/AbstractDataBackedRestAPIIntegrationTest.java +++ b/data/beaconrestapi/src/integration-test/java/tech/pegasys/teku/beaconrestapi/AbstractDataBackedRestAPIIntegrationTest.java @@ -72,6 +72,7 @@ import tech.pegasys.teku.statetransition.attestation.AggregatingAttestationPool; import tech.pegasys.teku.statetransition.attestation.AttestationManager; import tech.pegasys.teku.statetransition.blobs.BlobSidecarManager; +import tech.pegasys.teku.statetransition.blobs.BlobSidecarPool; import tech.pegasys.teku.statetransition.block.BlockManager; import tech.pegasys.teku.statetransition.forkchoice.ForkChoice; import tech.pegasys.teku.statetransition.forkchoice.MergeTransitionBlockValidator; @@ -225,6 +226,7 @@ private void setupAndStartRestAPI(BeaconRestApiConfig config) { .syncService(syncService) .validatorApiChannel(validatorApiChannel) .blockManager(blockManager) + .blobSidecarPool(BlobSidecarPool.NOOP) .attestationManager(attestationManager) .activeValidatorChannel(activeValidatorChannel) .attestationPool(attestationPool) diff --git a/data/beaconrestapi/src/integration-test/resources/tech/pegasys/teku/beaconrestapi/beacon/paths/_eth_v1_events.json b/data/beaconrestapi/src/integration-test/resources/tech/pegasys/teku/beaconrestapi/beacon/paths/_eth_v1_events.json index fce498eb1b8..4fcf93bc570 100644 --- a/data/beaconrestapi/src/integration-test/resources/tech/pegasys/teku/beaconrestapi/beacon/paths/_eth_v1_events.json +++ b/data/beaconrestapi/src/integration-test/resources/tech/pegasys/teku/beaconrestapi/beacon/paths/_eth_v1_events.json @@ -9,7 +9,7 @@ "in" : "query", "schema" : { "type" : "string", - "description" : "Event types to subscribe to. Available values include: [`head`, `finalized_checkpoint`, `chain_reorg`, `block`, `attestation`, `voluntary_exit`, `contribution_and_proof`]\n\n", + "description" : "Event types to subscribe to. Available values include: [`head`, `finalized_checkpoint`, `chain_reorg`, `block`, `attestation`, `voluntary_exit`, `contribution_and_proof`, `blob_sidecar`]\n\n", "example" : "head" } } ], From 12e4620ce7297adbf2798530b88d707b8b141659 Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Mon, 2 Oct 2023 09:10:52 +0200 Subject: [PATCH 4/4] add final and a new test --- .../v1/events/EventSubscriptionManager.java | 2 +- .../util/BlobSidecarPoolImplTest.java | 24 +++++++++++++++++-- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java b/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java index d83d9c14f6a..a9a166badd5 100644 --- a/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java +++ b/data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/events/EventSubscriptionManager.java @@ -54,7 +54,7 @@ public class EventSubscriptionManager implements ChainHeadChannel, FinalizedCheckpointChannel { private static final Logger LOG = LogManager.getLogger(); - private Spec spec; + private final Spec spec; private final ConfigProvider configProvider; private final ChainDataProvider provider; private final AsyncRunner asyncRunner; diff --git a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/util/BlobSidecarPoolImplTest.java b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/util/BlobSidecarPoolImplTest.java index aa4598f7cf5..a4b1407fae1 100644 --- a/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/util/BlobSidecarPoolImplTest.java +++ b/ethereum/statetransition/src/test/java/tech/pegasys/teku/statetransition/util/BlobSidecarPoolImplTest.java @@ -124,7 +124,7 @@ public void onNewBlock_addTrackerWithBlock() { } @Test - public void onNewBlobSidecar_addTrackerWithBlobSidecar() { + public void onNewBlobSidecar_addTrackerWithBlobSidecarIgnoringDuplicates() { final BlobSidecar blobSidecar = dataStructureUtil.createRandomBlobSidecarBuilder().slot(currentSlot).build(); @@ -142,6 +142,26 @@ public void onNewBlobSidecar_addTrackerWithBlobSidecar() { assertBlobSidecarsTrackersCount(1); } + @Test + public void onNewBlobSidecar_shouldIgnoreDuplicates() { + final BlobSidecar blobSidecar = + dataStructureUtil.createRandomBlobSidecarBuilder().slot(currentSlot).build(); + + blobSidecarPool.onNewBlobSidecar(blobSidecar); + blobSidecarPool.onNewBlobSidecar(blobSidecar); + + assertThat(blobSidecarPool.containsBlobSidecar(blobIdentifierFromBlobSidecar(blobSidecar))) + .isTrue(); + assertThat(requiredBlockRootEvents).isEmpty(); + assertThat(requiredBlockRootDroppedEvents).isEmpty(); + assertThat(requiredBlobSidecarEvents).isEmpty(); + assertThat(requiredBlobSidecarDroppedEvents).isEmpty(); + assertThat(newBlobSidecarEvents).containsExactly(blobSidecar); + + assertBlobSidecarsCount(1); + assertBlobSidecarsTrackersCount(1); + } + @Test public void onNewBlock_shouldIgnorePreDenebBlocks() { final Spec spec = TestSpecFactory.createMainnetCapella(); @@ -342,7 +362,7 @@ public void onCompletedBlockAndBlobSidecars_shouldNotTriggerFetch() { @Test public void - getOrCreateBlockblockBlobSidecarsTracker_createATrackerWithBlockSetIgnoringHistoricalTolerance() { + getOrCreateBlocBlobSidecarsTracker_createATrackerWithBlockSetIgnoringHistoricalTolerance() { final UInt64 slot = currentSlot.minus(historicalTolerance).minus(UInt64.ONE); final SignedBeaconBlock block = dataStructureUtil.randomSignedBeaconBlock(slot);