Merge branch 'master' into das-electra

CHANGELOG.md

@@ -9,7 +9,7 @@
 ### Breaking Changes
 
 ### Additions and Improvements
-- Clean up old beacon states when switching from ARCHIVE to PRUNE or MINIMAL data storage mode
+- improve block publishing performance, especially relevant with locally produced blocks
 
 ### Bug Fixes
- - Fixed a block production issue for Validator Client (24.10.0 to 24.10.2 teku VC), where required headers were not provided for JSON payloads. Default SSZ block production was unaffected. The required header is in the beacon-api spec but was not updated in all places for the VC.
+- Added a startup script for unix systems to ensure that when jemalloc is installed the script sets the LD_PRELOAD environment variable to the use the jemalloc library

acceptance-tests/src/testFixtures/java/tech/pegasys/teku/test/acceptance/dsl/TekuNodeConfigBuilder.java

@@ -14,6 +14,7 @@
 package tech.pegasys.teku.test.acceptance.dsl;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.assertj.core.api.Assertions.assertThat;
 import static tech.pegasys.teku.test.acceptance.dsl.Node.DATA_PATH;
 import static tech.pegasys.teku.test.acceptance.dsl.Node.JWT_SECRET_FILE_PATH;
 import static tech.pegasys.teku.test.acceptance.dsl.Node.METRICS_PORT;
@@ -176,7 +177,7 @@ public TekuNodeConfigBuilder withTrustedSetupFromClasspath(final String trustedS
     LOG.debug("Xtrusted-setup={}", TRUSTED_SETUP_FILE);
     configMap.put("Xtrusted-setup", TRUSTED_SETUP_FILE);
     final URL trustedSetupResource = Eth2NetworkConfiguration.class.getResource(trustedSetup);
-    assert trustedSetupResource != null;
+    assertThat(trustedSetupResource).isNotNull();
     configFileMap.put(copyToTmpFile(trustedSetupResource), TRUSTED_SETUP_FILE);
     return this;
   }

acceptance-tests/src/testFixtures/java/tech/pegasys/teku/test/acceptance/dsl/tools/ValidatorKeysApi.java

@@ -255,9 +255,7 @@ private String addRemoteValidators(final List<BLSPublicKey> publicKeys, final St
       throws IOException {
 
     List<Map<String, String>> requestPayload =
-        publicKeys.stream()
-            .map(k -> remotePostRequestBody(k, signerUrl))
-            .collect(Collectors.toList());
+        publicKeys.stream().map(k -> remotePostRequestBody(k, signerUrl)).toList();
     final String body = objectMapper.writeValueAsString(Map.of("remote_keys", requestPayload));
 
     final String result = httpClient.post(validatorUri.get(), REMOTE_KEYS_URL, body, authHeaders());

acceptance-tests/src/testFixtures/java/tech/pegasys/teku/test/acceptance/dsl/tools/deposits/DepositGenerator.java

@@ -14,7 +14,6 @@
 package tech.pegasys.teku.test.acceptance.dsl.tools.deposits;
 
 import java.util.List;
-import java.util.stream.Collectors;
 import org.web3j.crypto.Credentials;
 import org.web3j.protocol.core.methods.response.TransactionReceipt;
 import tech.pegasys.teku.ethereum.execution.types.Eth1Address;
@@ -40,9 +39,7 @@ public SafeFuture<Void> sendDeposits(final ValidatorKeystores validators) {
     return SafeFuture.of(
         () -> {
           final List<SafeFuture<TransactionReceipt>> transactionReceipts =
-              validators.getValidatorKeys().stream()
-                  .map(this::sendDeposit)
-                  .collect(Collectors.toList());
+              validators.getValidatorKeys().stream().map(this::sendDeposit).toList();
           return SafeFuture.allOf(transactionReceipts.toArray(SafeFuture[]::new));
         });
   }

acceptance-tests/src/testFixtures/java/tech/pegasys/teku/test/acceptance/dsl/tools/deposits/ValidatorKeys.java

@@ -22,7 +22,7 @@
 public class ValidatorKeys {
   private final BLSKeyPair validatorKey;
 
-  private boolean locked = false;
+  private final boolean locked;
   private final Bytes32 withdrawalCredentials;
   private final Optional<BLSKeyPair> withdrawalKey;
 

beacon/pow/src/test/java/tech/pegasys/teku/beacon/pow/DepositProcessingControllerTest.java

@@ -53,7 +53,7 @@ public class DepositProcessingControllerTest {
   private DepositProcessingController depositProcessingController;
 
   private void createDepositProcessingController(final Consumer<SpecConfigBuilder> configModifier) {
-    final SpecConfig config = SpecConfigLoader.loadConfig("minimal", configModifier);
+    final SpecConfig config = SpecConfigLoader.loadConfig("minimal", configModifier).specConfig();
     depositProcessingController =
         new DepositProcessingController(
             config,

beacon/pow/src/test/java/tech/pegasys/teku/beacon/pow/Eth1DepositManagerTest.java

@@ -56,9 +56,10 @@ class Eth1DepositManagerTest {
   private static final int MIN_GENESIS_BLOCK_TIMESTAMP = 10_000;
   private final SpecConfig config =
       SpecConfigLoader.loadConfig(
-          "minimal",
-          builder ->
-              builder.minGenesisTime(UInt64.valueOf(10_300)).genesisDelay(UInt64.valueOf(300)));
+              "minimal",
+              builder ->
+                  builder.minGenesisTime(UInt64.valueOf(10_300)).genesisDelay(UInt64.valueOf(300)))
+          .specConfig();
   private final Spec spec = TestSpecFactory.createMinimalBellatrix();
   private final DataStructureUtil dataStructureUtil = new DataStructureUtil(spec);
 

beacon/pow/src/test/java/tech/pegasys/teku/beacon/pow/MinimumGenesisTimeBlockFinderTest.java

@@ -38,7 +38,8 @@ public class MinimumGenesisTimeBlockFinderTest {
 
   // Setup so genesis time for a block will be blockTime + 2
   private final SpecConfig config =
-      SpecConfigLoader.loadConfig("minimal", builder -> builder.genesisDelay(UInt64.valueOf(2)));
+      SpecConfigLoader.loadConfig("minimal", builder -> builder.genesisDelay(UInt64.valueOf(2)))
+          .specConfig();
   private final Eth1Provider eth1Provider = mock(Eth1Provider.class);
   private final StubAsyncRunner asyncRunner = new StubAsyncRunner();
 
@@ -303,7 +304,8 @@ private void withMinGenesisTime(
       final long minGenesisTime, final Optional<UInt64> eth1DepositContractDeployBlock) {
     final SpecConfig config =
         SpecConfigLoader.loadConfig(
-            "minimal", builder -> builder.minGenesisTime(UInt64.valueOf(minGenesisTime)));
+                "minimal", builder -> builder.minGenesisTime(UInt64.valueOf(minGenesisTime)))
+            .specConfig();
     minimumGenesisTimeBlockFinder =
         new MinimumGenesisTimeBlockFinder(config, eth1Provider, eth1DepositContractDeployBlock);
   }

beacon/pow/src/test/java/tech/pegasys/teku/beacon/pow/Web3jEth1ProviderTest.java

@@ -170,13 +170,13 @@ void needsToBeValidatedChecks() {
 
     // advance less then required
     timeProvider.advanceTimeBySeconds(
-        Constants.ETH1_INVALID_ENDPOINT_CHECK_INTERVAL.getSeconds() - 10);
+        Constants.ETH1_INVALID_ENDPOINT_CHECK_INTERVAL.toSeconds() - 10);
 
     // just failed, no need to validate
     assertThat(provider.needsToBeValidated()).isFalse();
 
     // advance to go after
-    timeProvider.advanceTimeBySeconds(Constants.ETH1_INVALID_ENDPOINT_CHECK_INTERVAL.getSeconds());
+    timeProvider.advanceTimeBySeconds(Constants.ETH1_INVALID_ENDPOINT_CHECK_INTERVAL.toSeconds());
 
     // after the interval needs to be validated
     assertThat(provider.needsToBeValidated()).isTrue();
@@ -186,8 +186,7 @@ void needsToBeValidatedChecks() {
     // just validated, no need to validate
     assertThat(provider.needsToBeValidated()).isFalse();
 
-    timeProvider.advanceTimeBySeconds(
-        Constants.ETH1_VALID_ENDPOINT_CHECK_INTERVAL.getSeconds() + 1);
+    timeProvider.advanceTimeBySeconds(Constants.ETH1_VALID_ENDPOINT_CHECK_INTERVAL.toSeconds() + 1);
 
     // after the interval needs to be validated
     assertThat(provider.needsToBeValidated()).isTrue();
@@ -198,13 +197,13 @@ void needsToBeValidatedChecks() {
 
     // advance less then required
     timeProvider.advanceTimeBySeconds(
-        Constants.ETH1_FAILED_ENDPOINT_CHECK_INTERVAL.getSeconds() - 10);
+        Constants.ETH1_FAILED_ENDPOINT_CHECK_INTERVAL.toSeconds() - 10);
 
     // just failed a call, no need to validate
     assertThat(provider.needsToBeValidated()).isFalse();
 
     // advance to go after
-    timeProvider.advanceTimeBySeconds(Constants.ETH1_FAILED_ENDPOINT_CHECK_INTERVAL.getSeconds());
+    timeProvider.advanceTimeBySeconds(Constants.ETH1_FAILED_ENDPOINT_CHECK_INTERVAL.toSeconds());
 
     // after the interval needs to be validated
     assertThat(provider.needsToBeValidated()).isTrue();

beacon/pow/src/test/java/tech/pegasys/teku/beacon/pow/api/Eth1DataCachePeriodCalculatorTest.java

@@ -21,7 +21,7 @@
 import tech.pegasys.teku.spec.config.SpecConfigLoader;
 
 class Eth1DataCachePeriodCalculatorTest {
-  private final SpecConfig config = SpecConfigLoader.loadConfig("minimal");
+  private final SpecConfig config = SpecConfigLoader.loadConfig("minimal").specConfig();
 
   @Test
   void shouldCalculateCachePeriodForMinimalConstantsFromFollowDistance() {

beacon/sync/src/main/java/tech/pegasys/teku/beacon/sync/forward/multipeer/chains/SyncSourceFactory.java

@@ -45,7 +45,8 @@ public SyncSource getOrCreateSyncSource(final Eth2Peer peer, final Spec spec) {
     // Limit request rate to just a little under what we'd accept
     final int maxBlocksPerMinute = this.maxBlocksPerMinute - batchSize - 1;
     final Optional<Integer> maxBlobSidecarsPerMinute =
-        spec.getMaxBlobsPerBlock().map(maxBlobsPerBlock -> maxBlocksPerMinute * maxBlobsPerBlock);
+        spec.getMaxBlobsPerBlockForHighestMilestone()
+            .map(maxBlobsPerBlock -> maxBlocksPerMinute * maxBlobsPerBlock);
     final Optional<Integer> maxDataColumnSidecarsPerMinute =
         spec.getNumberOfDataColumns()
             .map(dataColumnsPerBlock -> maxBlocksPerMinute * dataColumnsPerBlock.intValue());

beacon/sync/src/main/java/tech/pegasys/teku/beacon/sync/gossip/blobs/RecentBlobSidecarsFetchService.java

@@ -60,7 +60,8 @@ public static RecentBlobSidecarsFetchService create(
       final FetchTaskFactory fetchTaskFactory,
       final Spec spec) {
     final int maxConcurrentRequests =
-        RecentBlocksFetchService.MAX_CONCURRENT_REQUESTS * spec.getMaxBlobsPerBlock().orElse(1);
+        RecentBlocksFetchService.MAX_CONCURRENT_REQUESTS
+            * spec.getMaxBlobsPerBlockForHighestMilestone().orElse(1);
     return new RecentBlobSidecarsFetchService(
         asyncRunner,
         blockBlobSidecarsTrackersPool,

beacon/sync/src/testFixtures/java/tech/pegasys/teku/beacon/sync/SyncingNodeManager.java

@@ -84,6 +84,7 @@ public class SyncingNodeManager {
   private final BlockGossipChannel blockGossipChannel;
 
   private SyncingNodeManager(
+      final AsyncRunner asyncRunner,
       final EventChannels eventChannels,
       final RecentChainData recentChainData,
       final BeaconChainUtil chainUtil,
@@ -94,7 +95,7 @@ private SyncingNodeManager(
     this.chainUtil = chainUtil;
     this.eth2P2PNetwork = eth2P2PNetwork;
     this.syncService = syncService;
-    this.blockGossipChannel = eventChannels.getPublisher(BlockGossipChannel.class);
+    this.blockGossipChannel = eventChannels.getPublisher(BlockGossipChannel.class, asyncRunner);
   }
 
   @SuppressWarnings("FutureReturnValueIgnored")
@@ -145,6 +146,7 @@ public static SyncingNodeManager create(
 
     final BlockImporter blockImporter =
         new BlockImporter(
+            asyncRunner,
             spec,
             receivedBlockEventsChannelPublisher,
             recentChainData,
@@ -215,7 +217,7 @@ public static SyncingNodeManager create(
     syncService.start().join();
 
     return new SyncingNodeManager(
-        eventChannels, recentChainData, chainUtil, eth2P2PNetwork, syncService);
+        asyncRunner, eventChannels, recentChainData, chainUtil, eth2P2PNetwork, syncService);
   }
 
   public SafeFuture<Peer> connect(final SyncingNodeManager peer) {
@@ -250,6 +252,6 @@ public void setSlot(final UInt64 slot) {
   }
 
   public void gossipBlock(final SignedBeaconBlock block) {
-    blockGossipChannel.publishBlock(block);
+    blockGossipChannel.publishBlock(block).ifExceptionGetsHereRaiseABug();
   }
 }