Feature/fleet mode rebase (hyperledger#6641)

* fleet mode squash commit rebase

Signed-off-by: garyschulte <garyschulte@gmail.com>

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java

@@ -180,11 +180,10 @@ public void startNode(final BesuNode node) {
             .from(node.getMiningParameters())
             .transactionSelectionService(transactionSelectionServiceImpl)
             .build();
-    commonPluginConfiguration.init(
-        dataDir,
-        dataDir.resolve(DATABASE_PATH),
-        node.getDataStorageConfiguration(),
-        miningParameters);
+    commonPluginConfiguration
+        .init(dataDir, dataDir.resolve(DATABASE_PATH), node.getDataStorageConfiguration())
+        .withMiningParameters(miningParameters);
+
     final BesuPluginContextImpl besuPluginContext =
         besuPluginContextMap.computeIfAbsent(
             node,

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/privacy/PrivacyNode.java

@@ -274,7 +274,9 @@ public NodeConfiguration getConfiguration() {
   private PrivacyStorageProvider createKeyValueStorageProvider(
       final Path dataLocation, final Path dbLocation) {
     final var besuConfiguration = new BesuConfigurationImpl();
-    besuConfiguration.init(dataLocation, dbLocation, null, besuConfig.getMiningParameters());
+    besuConfiguration
+        .init(dataLocation, dbLocation, null)
+        .withMiningParameters(besuConfig.getMiningParameters());
     return new PrivacyKeyValueStorageProviderBuilder()
         .withStorageFactory(
             new RocksDBKeyValuePrivacyStorageFactory(

besu/src/main/java/org/hyperledger/besu/Runner.java

@@ -25,6 +25,7 @@
 import org.hyperledger.besu.ethereum.chain.Blockchain;
 import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolEvictionService;
 import org.hyperledger.besu.ethereum.p2p.network.NetworkRunner;
+import org.hyperledger.besu.ethereum.p2p.network.P2PNetwork;
 import org.hyperledger.besu.ethereum.stratum.StratumServer;
 import org.hyperledger.besu.ethstats.EthStatsService;
 import org.hyperledger.besu.metrics.MetricsService;
@@ -422,6 +423,15 @@ Optional<EnodeURL> getLocalEnode() {
     return networkRunner.getNetwork().getLocalEnode();
   }
 
+  /**
+   * get P2PNetwork service.
+   *
+   * @return p2p network service.
+   */
+  public P2PNetwork getP2PNetwork() {
+    return networkRunner.getNetwork();
+  }
+
   @FunctionalInterface
   private interface SynchronousShutdown {
     /**

besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java

@@ -172,21 +172,29 @@
 import org.hyperledger.besu.plugin.services.exception.StorageException;
 import org.hyperledger.besu.plugin.services.metrics.MetricCategory;
 import org.hyperledger.besu.plugin.services.metrics.MetricCategoryRegistry;
+import org.hyperledger.besu.plugin.services.p2p.P2PService;
+import org.hyperledger.besu.plugin.services.rlp.RlpConverterService;
 import org.hyperledger.besu.plugin.services.securitymodule.SecurityModule;
 import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;
 import org.hyperledger.besu.plugin.services.storage.PrivacyKeyValueStorageFactory;
 import org.hyperledger.besu.plugin.services.storage.rocksdb.RocksDBPlugin;
+import org.hyperledger.besu.plugin.services.sync.SynchronizationService;
+import org.hyperledger.besu.plugin.services.transactionpool.TransactionPoolService;
 import org.hyperledger.besu.services.BesuConfigurationImpl;
 import org.hyperledger.besu.services.BesuEventsImpl;
 import org.hyperledger.besu.services.BesuPluginContextImpl;
 import org.hyperledger.besu.services.BlockchainServiceImpl;
+import org.hyperledger.besu.services.P2PServiceImpl;
 import org.hyperledger.besu.services.PermissioningServiceImpl;
 import org.hyperledger.besu.services.PicoCLIOptionsImpl;
 import org.hyperledger.besu.services.PrivacyPluginServiceImpl;
+import org.hyperledger.besu.services.RlpConverterServiceImpl;
 import org.hyperledger.besu.services.RpcEndpointServiceImpl;
 import org.hyperledger.besu.services.SecurityModuleServiceImpl;
 import org.hyperledger.besu.services.StorageServiceImpl;
+import org.hyperledger.besu.services.SynchronizationServiceImpl;
 import org.hyperledger.besu.services.TraceServiceImpl;
+import org.hyperledger.besu.services.TransactionPoolServiceImpl;
 import org.hyperledger.besu.services.TransactionPoolValidatorServiceImpl;
 import org.hyperledger.besu.services.TransactionSelectionServiceImpl;
 import org.hyperledger.besu.services.TransactionSimulationServiceImpl;
@@ -1161,12 +1169,15 @@ public void run() {
       final var runner = buildRunner();
       runner.startExternalServices();
 
-      startPlugins();
+      startPlugins(runner);
       validatePluginOptions();
       setReleaseMetrics();
       preSynchronization();
 
       runner.startEthereumMainLoop();
+
+      besuPluginContext.afterExternalServicesMainLoop();
+
       runner.awaitStop();
 
     } catch (final Exception e) {
@@ -1327,7 +1338,7 @@ private Runner buildRunner() {
         pidPath);
   }
 
-  private void startPlugins() {
+  private void startPlugins(final Runner runner) {
     blockchainServiceImpl.init(
         besuController.getProtocolContext(), besuController.getProtocolSchedule());
     transactionSimulationServiceImpl.init(
@@ -1348,6 +1359,26 @@ private void startPlugins() {
             besuController.getProtocolContext().getBadBlockManager()));
     besuPluginContext.addService(MetricsSystem.class, getMetricsSystem());
 
+    besuPluginContext.addService(BlockchainService.class, blockchainServiceImpl);
+
+    besuPluginContext.addService(
+        SynchronizationService.class,
+        new SynchronizationServiceImpl(
+            besuController.getProtocolContext(),
+            besuController.getProtocolSchedule(),
+            besuController.getSyncState(),
+            besuController.getProtocolContext().getWorldStateArchive()));
+
+    besuPluginContext.addService(P2PService.class, new P2PServiceImpl(runner.getP2PNetwork()));
+
+    besuPluginContext.addService(
+        TransactionPoolService.class,
+        new TransactionPoolServiceImpl(besuController.getTransactionPool()));
+
+    besuPluginContext.addService(
+        RlpConverterService.class,
+        new RlpConverterServiceImpl(besuController.getProtocolSchedule()));
+
     besuPluginContext.addService(
         TraceService.class,
         new TraceServiceImpl(
@@ -1653,11 +1684,11 @@ private void validateRpcWsOptions() {
   private void validateChainDataPruningParams() {
     if (unstableChainPruningOptions.getChainDataPruningEnabled()
         && unstableChainPruningOptions.getChainDataPruningBlocksRetained()
-            < ChainPruningOptions.DEFAULT_CHAIN_DATA_PRUNING_MIN_BLOCKS_RETAINED) {
+            < unstableChainPruningOptions.getChainDataPruningBlocksRetainedLimit()) {
       throw new ParameterException(
           this.commandLine,
           "--Xchain-pruning-blocks-retained must be >= "
-              + ChainPruningOptions.DEFAULT_CHAIN_DATA_PRUNING_MIN_BLOCKS_RETAINED);
+              + unstableChainPruningOptions.getChainDataPruningBlocksRetainedLimit());
     }
   }
 
@@ -1843,11 +1874,10 @@ public BesuController buildController() {
    * @return instance of BesuControllerBuilder
    */
   public BesuControllerBuilder getControllerBuilder() {
-    pluginCommonConfiguration.init(
-        dataDir(),
-        dataDir().resolve(DATABASE_PATH),
-        getDataStorageConfiguration(),
-        miningParametersSupplier.get());
+    pluginCommonConfiguration
+        .init(dataDir(), dataDir().resolve(DATABASE_PATH), getDataStorageConfiguration())
+        .withMiningParameters(getMiningParameters())
+        .withJsonRpcHttpOptions(jsonRpcHttpOptions);
     final KeyValueStorageProvider storageProvider = keyValueStorageProvider(keyValueStorageName);
     return controllerBuilderFactory
         .fromEthNetworkConfig(updateNetworkConfig(network), getDefaultSyncModeIfNotSet())

besu/src/main/java/org/hyperledger/besu/cli/options/stable/JsonRpcHttpOptions.java

@@ -475,6 +475,15 @@ public List<String> getRpcHttpApis() {
     return rpcHttpApis;
   }
 
+  /**
+   * Returns the host for RPC over HTTP.
+   *
+   * @return The port number
+   */
+  public String getRpcHttpHost() {
+    return rpcHttpHost;
+  }
+
   /**
    * Returns the port for RPC over HTTP.
    *

besu/src/main/java/org/hyperledger/besu/cli/options/unstable/ChainPruningOptions.java

@@ -28,10 +28,17 @@ public class ChainPruningOptions implements CLIOptions<ChainPrunerConfiguration>
   private static final String CHAIN_PRUNING_ENABLED_FLAG = "--Xchain-pruning-enabled";
   private static final String CHAIN_PRUNING_BLOCKS_RETAINED_FLAG =
       "--Xchain-pruning-blocks-retained";
+  private static final String CHAIN_PRUNING_BLOCKS_RETAINED_LIMIT_FLAG =
+      "--Xchain-pruning-blocks-retained-limit";
   private static final String CHAIN_PRUNING_FREQUENCY_FLAG = "--Xchain-pruning-frequency";
 
-  /** The constant DEFAULT_CHAIN_DATA_PRUNING_MIN_BLOCKS_RETAINED. */
-  public static final long DEFAULT_CHAIN_DATA_PRUNING_MIN_BLOCKS_RETAINED = 7200;
+  /**
+   * The "CHAIN_DATA_PRUNING_MIN_BLOCKS_RETAINED_LIMIT" field sets the minimum limit for the
+   * "CHAIN_DATA_PRUNING_MIN_BLOCKS_RETAINED" value. For most networks, the default value of this
+   * limit is the safest. Reducing this value requires careful consideration and understanding of
+   * the potential implications. Lowering this limit may have unintended side effects.
+   */
+  public static final long CHAIN_DATA_PRUNING_MIN_BLOCKS_RETAINED_LIMIT = 7200;
 
   /** The constant DEFAULT_CHAIN_DATA_PRUNING_FREQUENCY. */
   public static final int DEFAULT_CHAIN_DATA_PRUNING_FREQUENCY = 256;
@@ -47,11 +54,21 @@ public class ChainPruningOptions implements CLIOptions<ChainPrunerConfiguration>
       hidden = true,
       names = {CHAIN_PRUNING_BLOCKS_RETAINED_FLAG},
       description =
-          "The number of recent blocks for which to keep the chain data. Must be >= "
-              + DEFAULT_CHAIN_DATA_PRUNING_MIN_BLOCKS_RETAINED
+          "The number of recent blocks for which to keep the chain data. Should be >= "
+              + CHAIN_DATA_PRUNING_MIN_BLOCKS_RETAINED_LIMIT
+              + " (default: ${DEFAULT-VALUE})")
+  private final Long chainDataPruningBlocksRetained = CHAIN_DATA_PRUNING_MIN_BLOCKS_RETAINED_LIMIT;
+
+  @CommandLine.Option(
+      hidden = true,
+      names = {CHAIN_PRUNING_BLOCKS_RETAINED_LIMIT_FLAG},
+      description =
+          "Allows setting the limit below which no more blocks can be pruned. This prevents setting a value lower than this for "
+              + CHAIN_PRUNING_BLOCKS_RETAINED_FLAG
+              + ". This flag should be used with caution as reducing the limit may have unintended side effects."
               + " (default: ${DEFAULT-VALUE})")
-  private final Long chainDataPruningBlocksRetained =
-      DEFAULT_CHAIN_DATA_PRUNING_MIN_BLOCKS_RETAINED;
+  private final Long chainDataPruningBlocksRetainedLimit =
+      CHAIN_DATA_PRUNING_MIN_BLOCKS_RETAINED_LIMIT;
 
   @CommandLine.Option(
       hidden = true,
@@ -91,11 +108,21 @@ public Long getChainDataPruningBlocksRetained() {
     return chainDataPruningBlocksRetained;
   }
 
+  /**
+   * Get the configured number of retained blocks for chain pruning.
+   *
+   * @return the number of retained blocks
+   */
+  public Long getChainDataPruningBlocksRetainedLimit() {
+    return chainDataPruningBlocksRetainedLimit;
+  }
+
   @Override
   public ChainPrunerConfiguration toDomainObject() {
     return new ChainPrunerConfiguration(
         chainDataPruningEnabled,
         chainDataPruningBlocksRetained,
+        chainDataPruningBlocksRetainedLimit,
         chainDataPruningBlocksFrequency.getValue());
   }
 
@@ -106,6 +133,8 @@ public List<String> getCLIOptions() {
         chainDataPruningEnabled.toString(),
         CHAIN_PRUNING_BLOCKS_RETAINED_FLAG,
         chainDataPruningBlocksRetained.toString(),
+        CHAIN_PRUNING_BLOCKS_RETAINED_LIMIT_FLAG,
+        chainDataPruningBlocksRetainedLimit.toString(),
         CHAIN_PRUNING_FREQUENCY_FLAG,
         chainDataPruningBlocksFrequency.toString());
   }

besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java

@@ -610,16 +610,6 @@ public BesuController build() {
             blockchain, worldStateArchive, protocolSchedule, this::createConsensusContext);
     validateContext(protocolContext);
 
-    if (chainPrunerConfiguration.getChainPruningEnabled()) {
-      final ChainDataPruner chainDataPruner = createChainPruner(blockchainStorage);
-      blockchain.observeBlockAdded(chainDataPruner);
-      LOG.info(
-          "Chain data pruning enabled with recent blocks retained to be: "
-              + chainPrunerConfiguration.getChainPruningBlocksRetained()
-              + " and frequency to be: "
-              + chainPrunerConfiguration.getChainPruningBlocksFrequency());
-    }
-
     protocolSchedule.setPublicWorldStateArchiveForPrivacyBlockProcessor(
         protocolContext.getWorldStateArchive());
 
@@ -668,6 +658,16 @@ public BesuController build() {
     final boolean fullSyncDisabled = !SyncMode.isFullSync(syncConfig.getSyncMode());
     final SyncState syncState = new SyncState(blockchain, ethPeers, fullSyncDisabled, checkpoint);
 
+    if (chainPrunerConfiguration.getChainPruningEnabled()) {
+      final ChainDataPruner chainDataPruner = createChainPruner(blockchainStorage);
+      blockchain.observeBlockAdded(chainDataPruner);
+      LOG.info(
+          "Chain data pruning enabled with recent blocks retained to be: "
+              + chainPrunerConfiguration.getChainPruningBlocksRetained()
+              + " and frequency to be: "
+              + chainPrunerConfiguration.getChainPruningBlocksFrequency());
+    }
+
     final TransactionPool transactionPool =
         TransactionPoolFactory.createTransactionPool(
             protocolSchedule,

besu/src/main/java/org/hyperledger/besu/services/BesuConfigurationImpl.java

@@ -14,20 +14,26 @@
  */
 package org.hyperledger.besu.services;
 
+import org.hyperledger.besu.cli.options.stable.JsonRpcHttpOptions;
 import org.hyperledger.besu.datatypes.Wei;
 import org.hyperledger.besu.ethereum.core.MiningParameters;
 import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
 import org.hyperledger.besu.plugin.services.BesuConfiguration;
 import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;
 
 import java.nio.file.Path;
+import java.util.Optional;
 
 /** A concrete implementation of BesuConfiguration which is used in Besu plugin framework. */
 public class BesuConfigurationImpl implements BesuConfiguration {
   private Path storagePath;
   private Path dataPath;
   private DataStorageConfiguration dataStorageConfiguration;
-  private MiningParameters miningParameters;
+
+  // defaults
+  private MiningParameters miningParameters = MiningParameters.newDefault();
+  private Optional<String> rpcHttpHost = Optional.of("http://localhost");
+  private Optional<Integer> rpcHttpPort = Optional.of(8545);
 
   /** Default Constructor. */
   public BesuConfigurationImpl() {}
@@ -38,17 +44,49 @@ public BesuConfigurationImpl() {}
    * @param dataPath The Path representing data folder
    * @param storagePath The path representing storage folder
    * @param dataStorageConfiguration The data storage configuration
-   * @param miningParameters The mining parameters
+   * @return BesuConfigurationImpl instance
    */
-  public void init(
+  public BesuConfigurationImpl init(
       final Path dataPath,
       final Path storagePath,
-      final DataStorageConfiguration dataStorageConfiguration,
-      final MiningParameters miningParameters) {
+      final DataStorageConfiguration dataStorageConfiguration) {
     this.dataPath = dataPath;
     this.storagePath = storagePath;
     this.dataStorageConfiguration = dataStorageConfiguration;
+    return this;
+  }
+
+  /**
+   * Set the mining parameters
+   *
+   * @param miningParameters configured mining parameters
+   * @return BesuConfigurationImpl instance
+   */
+  public BesuConfigurationImpl withMiningParameters(final MiningParameters miningParameters) {
     this.miningParameters = miningParameters;
+    return this;
+  }
+
+  /**
+   * Set the RPC http options
+   *
+   * @param rpcHttpOptions configured rpc http options
+   * @return BesuConfigurationImpl instance
+   */
+  public BesuConfigurationImpl withJsonRpcHttpOptions(final JsonRpcHttpOptions rpcHttpOptions) {
+    this.rpcHttpHost = Optional.ofNullable(rpcHttpOptions.getRpcHttpHost());
+    this.rpcHttpPort = Optional.ofNullable(rpcHttpOptions.getRpcHttpPort());
+    return this;
+  }
+
+  @Override
+  public Optional<String> getRpcHttpHost() {
+    return rpcHttpHost;
+  }
+
+  @Override
+  public Optional<Integer> getRpcHttpPort() {
+    return rpcHttpPort;
   }
 
   @Override

besu/src/main/java/org/hyperledger/besu/services/BesuEventsImpl.java

@@ -114,6 +114,11 @@ public void removeBlockReorgListener(final long listenerIdentifier) {
     blockchain.removeObserver(listenerIdentifier);
   }
 
+  @Override
+  public long addInitialSyncCompletionListener(final InitialSyncCompletionListener listener) {
+    return syncState.subscribeCompletionReached(listener);
+  }
+
   @Override
   public long addTransactionAddedListener(final TransactionAddedListener listener) {
     return transactionPool.subscribePendingTransactions(listener::onTransactionAdded);