Merge branch 'master' into fix/active-validator-registration

eth-benchmark-tests/src/jmh/java/tech/pegasys/teku/benchmarks/EpochTransitionBenchmark.java

@@ -32,7 +32,7 @@
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.infra.Blackhole;
 import tech.pegasys.teku.benchmarks.gen.BlockIO;
-import tech.pegasys.teku.benchmarks.gen.BlsKeyPairIO;
+import tech.pegasys.teku.benchmarks.gen.KeyFileGenerator;
 import tech.pegasys.teku.benchmarks.util.CustomRunner;
 import tech.pegasys.teku.bls.BLSKeyPair;
 import tech.pegasys.teku.bls.BLSSignatureVerifier;
@@ -102,11 +102,8 @@ public void init() throws Exception {
             + "_validators_"
             + validatorsCount
             + ".ssz.gz";
-    String keysFile = "/bls-key-pairs/bls-key-pairs-400k-seed-0.txt.gz";
 
-    System.out.println("Generating keypairs from " + keysFile);
-    List<BLSKeyPair> validatorKeys =
-        BlsKeyPairIO.createReaderForResource(keysFile).readAll(validatorsCount);
+    final List<BLSKeyPair> validatorKeys = KeyFileGenerator.readValidatorKeys(validatorsCount);
 
     final BlockImportNotifications blockImportNotifications = mock(BlockImportNotifications.class);
     epochProcessor = spec.getGenesisSpec().getEpochProcessor();

eth-benchmark-tests/src/jmh/java/tech/pegasys/teku/benchmarks/ProcessSyncAggregateBenchmark.java

@@ -29,8 +29,7 @@
 import org.openjdk.jmh.annotations.State;
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
-import tech.pegasys.teku.benchmarks.gen.BlsKeyPairIO;
-import tech.pegasys.teku.benchmarks.gen.BlsKeyPairIO.Reader;
+import tech.pegasys.teku.benchmarks.gen.KeyFileGenerator;
 import tech.pegasys.teku.bls.BLSKeyPair;
 import tech.pegasys.teku.bls.BLSSignature;
 import tech.pegasys.teku.bls.BLSSignatureVerifier;
@@ -71,13 +70,8 @@ public void init() throws Exception {
             + "_validators_"
             + validatorsCount
             + ".ssz.gz";
-    String keysFile = "/bls-key-pairs/bls-key-pairs-400k-seed-0.txt.gz";
 
-    System.out.println("Generating keypairs from " + keysFile);
-    List<BLSKeyPair> validatorKeys;
-    try (final Reader reader = BlsKeyPairIO.createReaderForResource(keysFile)) {
-      validatorKeys = reader.readAll(validatorsCount);
-    }
+    final List<BLSKeyPair> validatorKeys = KeyFileGenerator.readValidatorKeys(validatorsCount);
 
     final DataStructureUtil dataStructureUtil = new DataStructureUtil(spec);
     state =

eth-benchmark-tests/src/jmh/java/tech/pegasys/teku/benchmarks/ProfilingRun.java

@@ -26,7 +26,7 @@
 import org.junit.jupiter.api.Test;
 import tech.pegasys.teku.benchmarks.gen.BlockIO;
 import tech.pegasys.teku.benchmarks.gen.BlockIO.Reader;
-import tech.pegasys.teku.benchmarks.gen.BlsKeyPairIO;
+import tech.pegasys.teku.benchmarks.gen.KeyFileGenerator;
 import tech.pegasys.teku.bls.BLSKeyPair;
 import tech.pegasys.teku.bls.BLSPublicKey;
 import tech.pegasys.teku.bls.BLSSignatureVerifier;
@@ -80,11 +80,7 @@ public void importBlocks() throws Exception {
             + validatorsCount
             + ".ssz.gz";
 
-    System.out.println("Generating keypairs...");
-
-    List<BLSKeyPair> validatorKeys =
-        BlsKeyPairIO.createReaderForResource("/bls-key-pairs/bls-key-pairs-200k-seed-0.txt.gz")
-            .readAll(validatorsCount);
+    List<BLSKeyPair> validatorKeys = KeyFileGenerator.readValidatorKeys(validatorsCount);
 
     BeaconState initialState =
         new GenesisStateBuilder()
@@ -165,20 +161,16 @@ public void importBlocksMemProfiling() throws Exception {
 
     AbstractBlockProcessor.depositSignatureVerifier = BLSSignatureVerifier.NO_OP;
 
-    int validatorsCount = 32 * 1024;
+    final int validatorsCount = 32 * 1024;
 
-    String blocksFile =
+    final String blocksFile =
         "/blocks/blocks_epoch_"
             + spec.getSlotsPerEpoch(UInt64.ZERO)
             + "_validators_"
             + validatorsCount
             + ".ssz.gz";
 
-    System.out.println("Generating keypairs...");
-
-    List<BLSKeyPair> validatorKeys =
-        BlsKeyPairIO.createReaderForResource("/bls-key-pairs/bls-key-pairs-200k-seed-0.txt.gz")
-            .readAll(validatorsCount);
+    final List<BLSKeyPair> validatorKeys = KeyFileGenerator.readValidatorKeys(validatorsCount);
 
     BeaconState initialState =
         new GenesisStateBuilder()
@@ -191,8 +183,8 @@ public void importBlocksMemProfiling() throws Exception {
     while (true) {
       final BlockImportNotifications blockImportNotifications =
           mock(BlockImportNotifications.class);
-      RecentChainData recentChainData = MemoryOnlyRecentChainData.create();
-      BeaconChainUtil localChain =
+      final RecentChainData recentChainData = MemoryOnlyRecentChainData.create();
+      final BeaconChainUtil localChain =
           BeaconChainUtil.create(spec, recentChainData, validatorKeys, false);
       recentChainData.initializeFromGenesis(initialState, UInt64.ZERO);
       initialState = null;

eth-benchmark-tests/src/jmh/java/tech/pegasys/teku/benchmarks/TransitionBenchmark.java

@@ -30,7 +30,7 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import tech.pegasys.teku.benchmarks.gen.BlockIO;
-import tech.pegasys.teku.benchmarks.gen.BlsKeyPairIO;
+import tech.pegasys.teku.benchmarks.gen.KeyFileGenerator;
 import tech.pegasys.teku.bls.BLSKeyPair;
 import tech.pegasys.teku.bls.BLSSignatureVerifier;
 import tech.pegasys.teku.infrastructure.async.eventthread.InlineEventThread;
@@ -82,11 +82,8 @@ public void init() throws Exception {
             + "_validators_"
             + validatorsCount
             + ".ssz.gz";
-    String keysFile = "/bls-key-pairs/bls-key-pairs-400k-seed-0.txt.gz";
 
-    System.out.println("Generating keypairs from " + keysFile);
-    List<BLSKeyPair> validatorKeys =
-        BlsKeyPairIO.createReaderForResource(keysFile).readAll(validatorsCount);
+    final List<BLSKeyPair> validatorKeys = KeyFileGenerator.readValidatorKeys(validatorsCount);
 
     final BlockImportNotifications blockImportNotifications = mock(BlockImportNotifications.class);
     wsValidator = WeakSubjectivityFactory.lenientValidator();

eth-benchmark-tests/src/jmh/java/tech/pegasys/teku/benchmarks/gen/Generator.java → eth-benchmark-tests/src/jmh/java/tech/pegasys/teku/benchmarks/gen/BlockArchiveGenerator.java

@@ -1,5 +1,5 @@
 /*
- * Copyright Consensys Software Inc., 2022
+ * Copyright Consensys Software Inc., 2023
  *
  * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
  * the License. You may obtain a copy of the License at
@@ -13,19 +13,12 @@
 
 package tech.pegasys.teku.benchmarks.gen;
 
-import java.io.File;
-import java.util.ArrayList;
 import java.util.Collections;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Optional;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
-import org.junit.jupiter.api.Test;
-import tech.pegasys.teku.benchmarks.gen.BlockIO.Writer;
 import tech.pegasys.teku.bls.BLSKeyPair;
 import tech.pegasys.teku.bls.BLSSignatureVerifier;
-import tech.pegasys.teku.bls.BLSTestUtil;
+import tech.pegasys.teku.infrastructure.time.SystemTimeProvider;
 import tech.pegasys.teku.infrastructure.unsigned.UInt64;
 import tech.pegasys.teku.spec.Spec;
 import tech.pegasys.teku.spec.TestSpecFactory;
@@ -39,49 +32,86 @@
 import tech.pegasys.teku.storage.client.MemoryOnlyRecentChainData;
 import tech.pegasys.teku.storage.client.RecentChainData;
 
-/**
- * Utility class for generating BLS keypairs and blocks files Test methods need to be run manually
- */
-public class Generator {
+public class BlockArchiveGenerator {
+  public static void main(String[] args) throws Exception {
+    int validatorCount = 32_768;
+    int epochCount = 50;
+    if (args.length == 2) {
+      // read cli positional args
+      try {
+        validatorCount = Integer.parseInt(args[0]);
+      } catch (Exception e) {
+        dieUsage(Optional.of("Failed to parse validatorCount: " + e.getMessage()));
+      }
+      try {
+        epochCount = Integer.parseInt(args[1]);
+      } catch (Exception e) {
+        dieUsage(Optional.of("Failed to parse epochCount: " + e.getMessage()));
+      }
+    } else if (args.length != 0) {
+      dieUsage(Optional.empty());
+    }
 
-  @Test
-  public void generateBlocks() throws Exception {
-    final Spec spec = TestSpecFactory.createMainnetAltair();
+    if (validatorCount < 1 || validatorCount > 3_276_800) {
+      dieUsage(
+          Optional.of(
+              "Expected validator count ("
+                  + validatorCount
+                  + ") to be in range 0 < validatorCount < 3_276_801"));
+    } else if (epochCount < 1 || epochCount > 100) {
+      dieUsage(
+          Optional.of(
+              "Expected epoch count (" + epochCount + ") to be in range 0 < epochCount < 100"));
+    }
+    System.out.println("Validator count: " + validatorCount);
+    System.out.println("Epochs: " + epochCount);
 
-    AbstractBlockProcessor.depositSignatureVerifier = BLSSignatureVerifier.NO_OP;
+    generateBlocks(validatorCount, epochCount);
+  }
 
-    System.out.println("Generating keypairs...");
-    int validatorsCount = 400000;
+  private static void dieUsage(final Optional<String> maybeContext) {
+    maybeContext.ifPresent(System.out::println);
+    System.out.println("Usage: blockArchiveGenerator <validatorCount> <epochs>");
+    System.exit(2);
+  }
 
-    List<BLSKeyPair> validatorKeys =
-        BlsKeyPairIO.createReaderForResource("/bls-key-pairs/bls-key-pairs-400k-seed-0.txt.gz")
-            .readAll(validatorsCount);
+  private static void generateBlocks(final int validatorsCount, final int epochLimit)
+      throws Exception {
+    final Spec spec = TestSpecFactory.createMainnetAltair();
 
-    System.out.println("Keypairs done.");
+    AbstractBlockProcessor.depositSignatureVerifier = BLSSignatureVerifier.NO_OP;
 
-    RecentChainData localStorage = MemoryOnlyRecentChainData.create(spec);
-    BeaconChainUtil localChain =
+    final List<BLSKeyPair> validatorKeys = KeyFileGenerator.readValidatorKeys(validatorsCount);
+    final RecentChainData localStorage = MemoryOnlyRecentChainData.create(spec);
+    final AttestationGenerator attestationGenerator = new AttestationGenerator(spec, validatorKeys);
+    final int slotsPerEpoch = spec.getGenesisSpecConfig().getSlotsPerEpoch();
+    final String blocksFile =
+        String.format(
+            "blocks_%sEpochs_%sBlocksPerEpoch_%sValidators.ssz.gz",
+            epochLimit, slotsPerEpoch, validatorsCount);
+    final BeaconChainUtil localChain =
         BeaconChainUtil.builder()
             .specProvider(spec)
             .recentChainData(localStorage)
             .validatorKeys(validatorKeys)
             .signDeposits(false)
             .build();
     localChain.initializeStorage();
-    AttestationGenerator attestationGenerator = new AttestationGenerator(spec, validatorKeys);
 
     UInt64 currentSlot = localStorage.getHeadSlot();
     List<Attestation> attestations = Collections.emptyList();
 
-    final int slotsPerEpoch = spec.getGenesisSpecConfig().getSlotsPerEpoch();
-    String blocksFile =
-        "blocks_epoch_" + slotsPerEpoch + "_validators_" + validatorsCount + ".ssz.gz";
+    System.out.printf(
+        "Generating blocks for %s epochs, %s slots per epoch.%n", epochLimit, slotsPerEpoch);
+
+    final SystemTimeProvider timeProvider = new SystemTimeProvider();
 
-    try (Writer writer = BlockIO.createFileWriter(blocksFile)) {
+    try (BlockIO.Writer writer = BlockIO.createFileWriter(blocksFile)) {
 
-      for (int j = 0; j < 50; j++) {
+      for (int j = 0; j < epochLimit; j++) {
+        System.out.println(" => Processing epoch " + j);
         for (int i = 0; i < slotsPerEpoch; i++) {
-          long s = System.currentTimeMillis();
+          final UInt64 slotStart = timeProvider.getTimeInMillis();
           currentSlot = currentSlot.plus(UInt64.ONE);
 
           final SignedBeaconBlock block =
@@ -101,58 +131,22 @@ public void generateBlocks() throws Exception {
                   : attestationGenerator.getAttestationsForSlot(postState, currentSlot);
 
           System.out.println(
-              "Processed: " + currentSlot + ", " + (System.currentTimeMillis() - s) + " ms");
+              "   -> Processed: "
+                  + currentSlot
+                  + ", "
+                  + (timeProvider.getTimeInMillis().minusMinZero(slotStart))
+                  + " ms");
         }
 
         Optional<BeaconState> bestState =
             localStorage.retrieveBlockState(localStorage.getBestBlockRoot().orElse(null)).join();
-        System.out.println("Epoch done: " + bestState);
+        final long epoch = j;
+        bestState.ifPresent(
+            beaconState ->
+                System.out.printf(
+                    " => Epoch done: %s, Best State slot: %s, state hash: %s%n",
+                    epoch, beaconState.getSlot(), beaconState.hashTreeRoot()));
       }
     }
   }
-
-  @Test
-  public void generateKeyPairs() throws Exception {
-    int randomSeed = 0;
-    int limitK = 400;
-    File outFile = new File("bls-key-pairs-" + limitK + "k-seed-" + randomSeed + ".txt");
-    Iterator<BLSKeyPair> keyPairIterator =
-        IntStream.range(randomSeed, randomSeed + Integer.MAX_VALUE)
-            .mapToObj(BLSTestUtil::randomKeyPair)
-            .iterator();
-
-    System.out.println("Generating keypairs...");
-    try (BlsKeyPairIO.Writer writer = BlsKeyPairIO.createWriter(outFile, keyPairIterator::next)) {
-      for (int i = 0; i < limitK; i++) {
-        writer.write(1024);
-        System.out.println("Generated " + (i + 1) + "K");
-      }
-    }
-
-    // check
-    try (BlsKeyPairIO.Reader reader = BlsKeyPairIO.createReaderForFile(outFile)) {
-      for (BLSKeyPair keyPair : reader.withLimit(10)) {
-        System.out.println(keyPair);
-      }
-    }
-  }
-
-  String getCommittees(final Spec spec, BeaconState state) {
-    UInt64 cnt = spec.getCommitteeCountPerSlot(state, spec.getCurrentEpoch(state));
-    List<List<Integer>> committees = new ArrayList<>();
-    for (UInt64 index = UInt64.ZERO; index.compareTo(cnt) < 0; index = index.plus(UInt64.ONE)) {
-
-      committees.add(spec.getBeaconCommittee(state, state.getSlot(), index));
-    }
-
-    return "["
-        + committees.stream()
-            .map(com -> com.stream().map(i -> "" + i).collect(Collectors.joining(",")))
-            .collect(Collectors.joining("],["))
-        + "]";
-  }
-
-  public static void main(String[] args) throws Exception {
-    new Generator().generateBlocks();
-  }
 }