Add experimental trie log pruning feature (hyperledger#6026)

- Toggled with --Xbonsai-trie-log-pruning-enabled

- Introduces TrieLogPruner which loads a limited number of trie logs on startup to preload the pruner queue, based on loadingLimit with a default value of 30,000 blocks (configured with --Xbonsai-trie-log-pruning-limit).

- Each time a trie log is persisted it is added to the pruner queue and then the pruner is run against the queue, which will prune trie logs associated with block numbers below the --Xbonsai-trie-log-retention-threshold (default 512).

- Once the retention threshold is reached, each prune run should just be a single trie log.

- Prune any orphaned trielogs that were created during block creation.

- Don't prune non-finalized blocks for PoS chains.

---------

Signed-off-by: Simon Dudley <simon.dudley@consensys.net>

besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java

@@ -294,7 +294,7 @@ public class BesuCommand implements DefaultCommandValues, Runnable {
   private final ChainPruningOptions unstableChainPruningOptions = ChainPruningOptions.create();
 
   // stable CLI options
-  private final DataStorageOptions dataStorageOptions = DataStorageOptions.create();
+  final DataStorageOptions dataStorageOptions = DataStorageOptions.create();
   private final EthstatsOptions ethstatsOptions = EthstatsOptions.create();
   private final NodePrivateKeyFileOption nodePrivateKeyFileOption =
       NodePrivateKeyFileOption.create();
@@ -1785,6 +1785,7 @@ private void validateOptions() {
     validateChainDataPruningParams();
     validatePostMergeCheckpointBlockRequirements();
     validateTransactionPoolOptions();
+    validateDataStorageOptions();
     p2pTLSConfigOptions.checkP2PTLSOptionsDependencies(logger, commandLine);
     pkiBlockCreationOptions.checkPkiBlockCreationOptionsDependencies(logger, commandLine);
   }
@@ -1793,6 +1794,10 @@ private void validateTransactionPoolOptions() {
     transactionPoolOptions.validate(commandLine, getActualGenesisConfigOptions());
   }
 
+  private void validateDataStorageOptions() {
+    dataStorageOptions.validate(commandLine);
+  }
+
   private void validateRequiredOptions() {
     commandLine
         .getCommandSpec()
@@ -3466,6 +3471,14 @@ private String generateConfigurationOverview() {
       builder.setHighSpecEnabled();
     }
 
+    if (dataStorageOptions.toDomainObject().getUnstable().getBonsaiTrieLogPruningEnabled()) {
+      builder.setTrieLogPruningEnabled();
+      builder.setTrieLogRetentionThreshold(
+          dataStorageOptions.toDomainObject().getUnstable().getBonsaiTrieLogRetentionThreshold());
+      builder.setTrieLogPruningLimit(
+          dataStorageOptions.toDomainObject().getUnstable().getBonsaiTrieLogPruningLimit());
+    }
+
     builder.setTxPoolImplementation(buildTransactionPoolConfiguration().getTxPoolImplementation());
     builder.setWorldStateUpdateMode(unstableEvmOptions.toDomainObject().worldUpdaterMode());
 

besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java

@@ -50,6 +50,9 @@ public class ConfigurationOverviewBuilder {
   private Collection<String> engineApis;
   private String engineJwtFilePath;
   private boolean isHighSpec = false;
+  private boolean isTrieLogPruningEnabled = false;
+  private long trieLogRetentionThreshold = 0;
+  private Integer trieLogPruningLimit = null;
   private TransactionPoolConfiguration.Implementation txPoolImplementation;
   private EvmConfiguration.WorldUpdaterMode worldStateUpdateMode;
   private Map<String, String> environment;
@@ -171,6 +174,38 @@ public ConfigurationOverviewBuilder setHighSpecEnabled() {
     return this;
   }
 
+  /**
+   * Sets trie log pruning enabled
+   *
+   * @return the builder
+   */
+  public ConfigurationOverviewBuilder setTrieLogPruningEnabled() {
+    isTrieLogPruningEnabled = true;
+    return this;
+  }
+
+  /**
+   * Sets trie log retention threshold
+   *
+   * @param threshold the number of blocks to retain trie logs for
+   * @return the builder
+   */
+  public ConfigurationOverviewBuilder setTrieLogRetentionThreshold(final long threshold) {
+    trieLogRetentionThreshold = threshold;
+    return this;
+  }
+
+  /**
+   * Sets trie log pruning limit
+   *
+   * @param limit the max number of blocks to load and prune trie logs for at startup
+   * @return the builder
+   */
+  public ConfigurationOverviewBuilder setTrieLogPruningLimit(final int limit) {
+    trieLogPruningLimit = limit;
+    return this;
+  }
+
   /**
    * Sets the txpool implementation in use.
    *
@@ -266,13 +301,25 @@ public String build() {
       lines.add("Engine JWT: " + engineJwtFilePath);
     }
 
+    lines.add("Using " + txPoolImplementation + " transaction pool implementation");
+
     if (isHighSpec) {
       lines.add("Experimental high spec configuration enabled");
     }
 
-    lines.add("Using " + txPoolImplementation + " transaction pool implementation");
     lines.add("Using " + worldStateUpdateMode + " worldstate update mode");
 
+    if (isTrieLogPruningEnabled) {
+      final StringBuilder trieLogPruningString = new StringBuilder();
+      trieLogPruningString
+          .append("Trie log pruning enabled: retention: ")
+          .append(trieLogRetentionThreshold);
+      if (trieLogPruningLimit != null) {
+        trieLogPruningString.append("; prune limit: ").append(trieLogPruningLimit);
+      }
+      lines.add(trieLogPruningString.toString());
+    }
+
     lines.add("");
     lines.add("Host:");
 

besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java

@@ -17,15 +17,21 @@
 package org.hyperledger.besu.cli.options.stable;
 
 import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD;
+import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_TRIE_LOG_PRUNING_ENABLED;
+import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_TRIE_LOG_PRUNING_LIMIT;
+import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_TRIE_LOG_RETENTION_THRESHOLD;
+import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD;
 
 import org.hyperledger.besu.cli.options.CLIOptions;
+import org.hyperledger.besu.cli.util.CommandLineUtils;
 import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
 import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat;
 import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration;
 
 import java.util.List;
 
 import org.apache.commons.lang3.StringUtils;
+import picocli.CommandLine;
 import picocli.CommandLine.Option;
 
 /** The Data storage CLI options. */
@@ -42,16 +48,41 @@ public class DataStorageOptions implements CLIOptions<DataStorageConfiguration>
       description =
           "Format to store trie data in.  Either FOREST or BONSAI (default: ${DEFAULT-VALUE}).",
       arity = "1")
-  private final DataStorageFormat dataStorageFormat = DataStorageFormat.FOREST;
+  private DataStorageFormat dataStorageFormat = DataStorageFormat.FOREST;
 
   @Option(
       names = {BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD, "--bonsai-maximum-back-layers-to-load"},
       paramLabel = "<LONG>",
       description =
           "Limit of historical layers that can be loaded with BONSAI (default: ${DEFAULT-VALUE}).",
       arity = "1")
-  private final Long bonsaiMaxLayersToLoad = DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD;
+  private Long bonsaiMaxLayersToLoad = DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD;
 
+  @CommandLine.ArgGroup(validate = false)
+  private final DataStorageOptions.Unstable unstableOptions = new Unstable();
+
+  static class Unstable {
+
+    @CommandLine.Option(
+        hidden = true,
+        names = {"--Xbonsai-trie-log-pruning-enabled"},
+        description = "Enable trie log pruning. (default: ${DEFAULT-VALUE})")
+    private boolean bonsaiTrieLogPruningEnabled = DEFAULT_BONSAI_TRIE_LOG_PRUNING_ENABLED;
+
+    @CommandLine.Option(
+        hidden = true,
+        names = {"--Xbonsai-trie-log-retention-threshold"},
+        description =
+            "The number of blocks for which to retain trie logs. (default: ${DEFAULT-VALUE})")
+    private long bonsaiTrieLogRetentionThreshold = DEFAULT_BONSAI_TRIE_LOG_RETENTION_THRESHOLD;
+
+    @CommandLine.Option(
+        hidden = true,
+        names = {"--Xbonsai-trie-log-pruning-limit"},
+        description =
+            "The max number of blocks to load and prune trie logs for at startup. (default: ${DEFAULT-VALUE})")
+    private int bonsaiTrieLogPruningLimit = DEFAULT_BONSAI_TRIE_LOG_PRUNING_LIMIT;
+  }
   /**
    * Create data storage options.
    *
@@ -61,21 +92,62 @@ public static DataStorageOptions create() {
     return new DataStorageOptions();
   }
 
+  /**
+   * Validates the data storage options
+   *
+   * @param commandLine the full commandLine to check all the options specified by the user
+   */
+  public void validate(final CommandLine commandLine) {
+    if (unstableOptions.bonsaiTrieLogPruningEnabled) {
+      if (unstableOptions.bonsaiTrieLogRetentionThreshold
+          < MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD) {
+        throw new CommandLine.ParameterException(
+            commandLine,
+            String.format(
+                "--Xbonsai-trie-log-retention-threshold minimum value is %d",
+                MINIMUM_BONSAI_TRIE_LOG_RETENTION_THRESHOLD));
+      }
+      if (unstableOptions.bonsaiTrieLogPruningLimit <= 0) {
+        throw new CommandLine.ParameterException(
+            commandLine,
+            String.format(
+                "--Xbonsai-trie-log-pruning-limit=%d must be greater than 0",
+                unstableOptions.bonsaiTrieLogPruningLimit));
+      }
+    }
+  }
+
+  static DataStorageOptions fromConfig(final DataStorageConfiguration domainObject) {
+    final DataStorageOptions dataStorageOptions = DataStorageOptions.create();
+    dataStorageOptions.dataStorageFormat = domainObject.getDataStorageFormat();
+    dataStorageOptions.bonsaiMaxLayersToLoad = domainObject.getBonsaiMaxLayersToLoad();
+    dataStorageOptions.unstableOptions.bonsaiTrieLogPruningEnabled =
+        domainObject.getUnstable().getBonsaiTrieLogPruningEnabled();
+    dataStorageOptions.unstableOptions.bonsaiTrieLogRetentionThreshold =
+        domainObject.getUnstable().getBonsaiTrieLogRetentionThreshold();
+    dataStorageOptions.unstableOptions.bonsaiTrieLogPruningLimit =
+        domainObject.getUnstable().getBonsaiTrieLogPruningLimit();
+
+    return dataStorageOptions;
+  }
+
   @Override
   public DataStorageConfiguration toDomainObject() {
     return ImmutableDataStorageConfiguration.builder()
         .dataStorageFormat(dataStorageFormat)
         .bonsaiMaxLayersToLoad(bonsaiMaxLayersToLoad)
+        .unstable(
+            ImmutableDataStorageConfiguration.Unstable.builder()
+                .bonsaiTrieLogPruningEnabled(unstableOptions.bonsaiTrieLogPruningEnabled)
+                .bonsaiTrieLogRetentionThreshold(unstableOptions.bonsaiTrieLogRetentionThreshold)
+                .bonsaiTrieLogPruningLimit(unstableOptions.bonsaiTrieLogPruningLimit)
+                .build())
         .build();
   }
 
   @Override
   public List<String> getCLIOptions() {
-    return List.of(
-        DATA_STORAGE_FORMAT,
-        dataStorageFormat.toString(),
-        BONSAI_STORAGE_FORMAT_MAX_LAYERS_TO_LOAD,
-        bonsaiMaxLayersToLoad.toString());
+    return CommandLineUtils.getCLIOptions(this, new DataStorageOptions());
   }
 
   /**

besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java

@@ -34,6 +34,7 @@
 import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateProvider;
 import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader;
 import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage;
+import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogPruner;
 import org.hyperledger.besu.ethereum.chain.Blockchain;
 import org.hyperledger.besu.ethereum.chain.BlockchainStorage;
 import org.hyperledger.besu.ethereum.chain.ChainDataPruner;
@@ -1066,14 +1067,30 @@ WorldStateArchive createWorldStateArchive(
       final Blockchain blockchain,
       final CachedMerkleTrieLoader cachedMerkleTrieLoader) {
     return switch (dataStorageConfiguration.getDataStorageFormat()) {
-      case BONSAI -> new BonsaiWorldStateProvider(
-          (BonsaiWorldStateKeyValueStorage) worldStateStorage,
-          blockchain,
-          Optional.of(dataStorageConfiguration.getBonsaiMaxLayersToLoad()),
-          cachedMerkleTrieLoader,
-          metricsSystem,
-          besuComponent.map(BesuComponent::getBesuPluginContext).orElse(null),
-          evmConfiguration);
+      case BONSAI -> {
+        final GenesisConfigOptions genesisConfigOptions = configOptionsSupplier.get();
+        final boolean isProofOfStake =
+            genesisConfigOptions.getTerminalTotalDifficulty().isPresent();
+        final TrieLogPruner trieLogPruner =
+            dataStorageConfiguration.getUnstable().getBonsaiTrieLogPruningEnabled()
+                ? new TrieLogPruner(
+                    (BonsaiWorldStateKeyValueStorage) worldStateStorage,
+                    blockchain,
+                    dataStorageConfiguration.getUnstable().getBonsaiTrieLogRetentionThreshold(),
+                    dataStorageConfiguration.getUnstable().getBonsaiTrieLogPruningLimit(),
+                    isProofOfStake)
+                : TrieLogPruner.noOpTrieLogPruner();
+        trieLogPruner.initialize();
+        yield new BonsaiWorldStateProvider(
+            (BonsaiWorldStateKeyValueStorage) worldStateStorage,
+            blockchain,
+            Optional.of(dataStorageConfiguration.getBonsaiMaxLayersToLoad()),
+            cachedMerkleTrieLoader,
+            metricsSystem,
+            besuComponent.map(BesuComponent::getBesuPluginContext).orElse(null),
+            evmConfiguration,
+            trieLogPruner);
+      }
       case FOREST -> {
         final WorldStatePreimageStorage preimageStorage =
             storageProvider.createWorldStatePreimageStorage();

besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java

@@ -33,6 +33,7 @@
 import org.hyperledger.besu.cli.config.EthNetworkConfig;
 import org.hyperledger.besu.cli.options.MiningOptions;
 import org.hyperledger.besu.cli.options.TransactionPoolOptions;
+import org.hyperledger.besu.cli.options.stable.DataStorageOptions;
 import org.hyperledger.besu.cli.options.stable.EthstatsOptions;
 import org.hyperledger.besu.cli.options.unstable.EthProtocolOptions;
 import org.hyperledger.besu.cli.options.unstable.MetricsCLIOptions;
@@ -568,6 +569,10 @@ public TransactionPoolOptions getTransactionPoolOptions() {
       return transactionPoolOptions;
     }
 
+    public DataStorageOptions getDataStorageOptions() {
+      return dataStorageOptions;
+    }
+
     public MetricsCLIOptions getMetricsCLIOptions() {
       return unstableMetricsCLIOptions;
     }

besu/src/test/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilderTest.java

@@ -148,6 +148,24 @@ void setHighSpecEnabled() {
     assertThat(highSpecEnabled).contains("Experimental high spec configuration enabled");
   }
 
+  @Test
+  void setTrieLogPruningEnabled() {
+    final String noTrieLogRetentionThresholdSet = builder.build();
+    assertThat(noTrieLogRetentionThresholdSet).doesNotContain("Trie log pruning enabled");
+
+    builder.setTrieLogPruningEnabled();
+    builder.setTrieLogRetentionThreshold(42);
+    String trieLogRetentionThresholdSet = builder.build();
+    assertThat(trieLogRetentionThresholdSet)
+        .contains("Trie log pruning enabled")
+        .contains("retention: 42");
+    assertThat(trieLogRetentionThresholdSet).doesNotContain("prune limit");
+
+    builder.setTrieLogPruningLimit(1000);
+    trieLogRetentionThresholdSet = builder.build();
+    assertThat(trieLogRetentionThresholdSet).contains("prune limit: 1000");
+  }
+
   @Test
   void setTxPoolImplementationLayered() {
     builder.setTxPoolImplementation(LAYERED);