refactor: Improved configuration of protocol updates (#816)

## Summary

Basically it's a big refactor of the rust side of the code, roughly:

* I've tried to re-organise things into clearer files/folders.
* I've done some renames and some slight reworkings to clarify certain
concepts:
* The `ProtocolUpdateFactory` trait has become a single
`ProtocolDefinitionResolver` implementation. This means we test our
production code rather than using various stubs, and I think it's also
easier to follow.
  * `ProtocolUpdate` has become `ProtocolUpdateTrigger`
* `UpgradableStateComputerConfig` has become
`ProtocolStateComputerConfig`
* I've introduced a `ProtocolUpdateDefinition` which gives access to a
static config and the ability to create a (boxed) `ProtocolUpdater`
(which is no longer responsible for defining the static config).
* I've added some more comments
* I've hard coded `babylon-genesis` as the defacto standard genesis
version everywhere. Attempting to use something else will fail. This was
just part of my move to decrease the number of moving parts /
opportunities for misconfiguration. (although still allows us to define
a different genesis name with a different config if we want!)

In terms of features/useful changes:
* Different network protocol configs now live in rust, and testnets now
attempt to run anemone at the start of epoch 3.
* `ProtocolConfig` now has a field to define some overrides for protocol
updating (which depends on the ProtocolUpdater). Don't worry test team -
we try to deserialize the old model so it shouldn't break your tests. We
should probably just make it versioned from some point to make the code
a little less hacky.
* Any protocol updates starting `test-` will inject a single empty test
transaction.
* Any protocol updates starting `custom-` will read flash transactions
from the raw overrides to inject; else will inject nothing - as
requested by @les-sheppard. I've pivoted across `test.rs` (which should
probably be renamed).

common/src/main/java/com/radixdlt/networks/Network.java

@@ -73,6 +73,8 @@
 
 @SuppressWarnings("OptionalUsedAsFieldOrParameterType")
 public enum Network {
+  // NOTE: For protocol configs, see e.g. `[..]/protocol_configs/mainnet_protocol_config.rs`
+  //       These are resolved by environment from RadixNodeModule.java.
 
   /// Public Facing Permanent Networks (0x00 - 0x09)
   // - mainnet

core-rust-bridge/src/main/java/com/radixdlt/protocol/ProtocolConfig.java

@@ -65,22 +65,50 @@
 package com.radixdlt.protocol;
 
 import com.google.common.collect.ImmutableList;
+import com.google.common.reflect.TypeToken;
+import com.radixdlt.rev2.NetworkDefinition;
+import com.radixdlt.sbor.NodeSborCodecs;
 import com.radixdlt.sbor.codec.CodecMap;
 import com.radixdlt.sbor.codec.StructCodec;
+import com.radixdlt.sbor.exceptions.SborDecodeException;
+import java.util.Map;
 
 public record ProtocolConfig(
-    String genesisProtocolVersion, ImmutableList<ProtocolUpdate> protocolUpdates) {
+    String genesisProtocolVersion,
+    ImmutableList<ProtocolUpdateTrigger> protocolUpdateTriggers,
+    Map<String, byte[]> rawProtocolUpdateContentOverrides) {
+
+  public static final String GENESIS_PROTOCOL_VERSION_NAME = "babylon-genesis";
+
+  public ProtocolConfig(ImmutableList<ProtocolUpdateTrigger> protocolUpdateTriggers) {
+    this(protocolUpdateTriggers, Map.of());
+  }
+
+  public ProtocolConfig(
+      ImmutableList<ProtocolUpdateTrigger> protocolUpdateTriggers,
+      Map<String, byte[]> rawProtocolUpdateContentOverrides) {
+    this(GENESIS_PROTOCOL_VERSION_NAME, protocolUpdateTriggers, rawProtocolUpdateContentOverrides);
+  }
+
   public static void registerCodec(CodecMap codecMap) {
     codecMap.register(
         ProtocolConfig.class,
         codecs -> StructCodec.fromRecordComponents(ProtocolConfig.class, codecs));
   }
 
+  public static ProtocolConfig sborDecode(byte[] encoded, String errorMessage) {
+    try {
+      return NodeSborCodecs.decode(encoded, NodeSborCodecs.resolveCodec(new TypeToken<>() {}));
+    } catch (SborDecodeException ex) {
+      throw new RuntimeException(errorMessage, ex);
+    }
+  }
+
   public static ProtocolConfig testingDefault() {
-    return new ProtocolConfig("testing-genesis", ImmutableList.of());
+    return new ProtocolConfig(ImmutableList.of());
   }
 
-  public static ProtocolConfig mainnet() {
-    return RustProtocolUpdate.mainnetConfig();
+  public static ProtocolConfig resolveForNetwork(NetworkDefinition networkDefinition) {
+    return RustProtocolUpdate.resolveForNetwork(networkDefinition);
   }
 }

core-rust-bridge/src/main/java/com/radixdlt/protocol/ProtocolUpdateEnactmentCondition.java

@@ -97,7 +97,7 @@ static ProtocolUpdateEnactmentCondition singleReadinessThresholdBetweenEpochs(
 
   static ProtocolUpdateEnactmentCondition readinessThresholdsBetweenEpochs(
       long minEpoch, long maxEpoch, ImmutableList<Tuple.Tuple2<Decimal, Long>> thresholds) {
-    return new EnactAtStartOfAnEpochIfSupportedAndWithinBounds(
+    return new EnactAtStartOfEpochIfValidatorsReady(
         UInt64.fromNonNegativeLong(minEpoch),
         UInt64.fromNonNegativeLong(maxEpoch),
         thresholds.stream()
@@ -109,10 +109,10 @@ static ProtocolUpdateEnactmentCondition readinessThresholdsBetweenEpochs(
   }
 
   static ProtocolUpdateEnactmentCondition unconditionallyAtEpoch(long epoch) {
-    return new EnactUnconditionallyAtEpoch(UInt64.fromNonNegativeLong(epoch));
+    return new EnactAtStartOfEpochUnconditionally(UInt64.fromNonNegativeLong(epoch));
   }
 
-  record EnactAtStartOfAnEpochIfSupportedAndWithinBounds(
+  record EnactAtStartOfEpochIfValidatorsReady(
       UInt64 lowerBoundInclusive,
       UInt64 upperBoundExclusive,
       ImmutableList<SignalledReadinessThreshold> readinessThresholds)
@@ -127,5 +127,6 @@ public static void registerCodec(CodecMap codecMap) {
     }
   }
 
-  record EnactUnconditionallyAtEpoch(UInt64 epoch) implements ProtocolUpdateEnactmentCondition {}
+  record EnactAtStartOfEpochUnconditionally(UInt64 epoch)
+      implements ProtocolUpdateEnactmentCondition {}
 }

core-rust-bridge/src/main/java/com/radixdlt/protocol/ProtocolUpdate.java → core-rust-bridge/src/main/java/com/radixdlt/protocol/ProtocolUpdateTrigger.java

@@ -67,15 +67,15 @@
 import com.radixdlt.sbor.codec.CodecMap;
 import com.radixdlt.sbor.codec.StructCodec;
 
-public record ProtocolUpdate(
+public record ProtocolUpdateTrigger(
     String nextProtocolVersion, ProtocolUpdateEnactmentCondition enactmentCondition) {
 
   public static final String ANEMONE = "anemone";
 
   public static void registerCodec(CodecMap codecMap) {
     codecMap.register(
-        ProtocolUpdate.class,
-        codecs -> StructCodec.fromRecordComponents(ProtocolUpdate.class, codecs));
+        ProtocolUpdateTrigger.class,
+        codecs -> StructCodec.fromRecordComponents(ProtocolUpdateTrigger.class, codecs));
   }
 
   public String readinessSignalName() {

core-rust-bridge/src/main/java/com/radixdlt/protocol/RustProtocolUpdate.java

@@ -64,12 +64,10 @@
 
 package com.radixdlt.protocol;
 
-import static com.radixdlt.lang.Tuple.tuple;
-
 import com.google.common.reflect.TypeToken;
 import com.radixdlt.environment.NodeRustEnvironment;
-import com.radixdlt.lang.Tuple;
 import com.radixdlt.monitoring.Metrics;
+import com.radixdlt.rev2.NetworkDefinition;
 import com.radixdlt.sbor.Natives;
 
 public final class RustProtocolUpdate {
@@ -96,21 +94,21 @@ private static native byte[] applyProtocolUpdate(
 
   private final Natives.Call1<String, ProtocolUpdateResult> applyProtocolUpdateFunc;
 
-  public static String readinessSignalName(ProtocolUpdate protocolUpdate) {
-    return readinessSignalNameFunc.call(protocolUpdate);
+  public static String readinessSignalName(ProtocolUpdateTrigger protocolUpdateTrigger) {
+    return readinessSignalNameFunc.call(protocolUpdateTrigger);
   }
 
-  private static final Natives.Call1<ProtocolUpdate, String> readinessSignalNameFunc =
+  private static final Natives.Call1<ProtocolUpdateTrigger, String> readinessSignalNameFunc =
       Natives.builder(RustProtocolUpdate::nativeReadinessSignalName).build(new TypeToken<>() {});
 
   private static native byte[] nativeReadinessSignalName(byte[] requestPayload);
 
-  public static ProtocolConfig mainnetConfig() {
-    return mainnetConfigFunc.call(tuple());
+  public static ProtocolConfig resolveForNetwork(NetworkDefinition networkDefinition) {
+    return resolveForNetworkFunc.call(networkDefinition);
   }
 
-  private static final Natives.Call1<Tuple.Tuple0, ProtocolConfig> mainnetConfigFunc =
-      Natives.builder(RustProtocolUpdate::nativeMainnetConfig).build(new TypeToken<>() {});
+  private static final Natives.Call1<NetworkDefinition, ProtocolConfig> resolveForNetworkFunc =
+      Natives.builder(RustProtocolUpdate::nativeResolveForNetwork).build(new TypeToken<>() {});
 
-  private static native byte[] nativeMainnetConfig(byte[] requestPayload);
+  private static native byte[] nativeResolveForNetwork(byte[] requestPayload);
 }

core-rust-bridge/src/main/java/com/radixdlt/sbor/NodeSborCodecs.java

@@ -125,7 +125,7 @@ public static void registerCodecsWithCodecMap(CodecMap codecMap) {
     LoggingConfig.registerCodec(codecMap);
     StateManagerConfig.registerCodec(codecMap);
     ProtocolConfig.registerCodec(codecMap);
-    ProtocolUpdate.registerCodec(codecMap);
+    ProtocolUpdateTrigger.registerCodec(codecMap);
     ProtocolUpdateEnactmentCondition.registerCodec(codecMap);
     ProtocolUpdateEnactmentCondition.SignalledReadinessThreshold.registerCodec(codecMap);
     ProtocolState.registerCodec(codecMap);

core-rust-bridge/src/main/java/com/radixdlt/statecomputer/ProtocolState.java

@@ -67,8 +67,8 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.radixdlt.lang.Tuple;
-import com.radixdlt.protocol.ProtocolUpdate;
 import com.radixdlt.protocol.ProtocolUpdateEnactmentCondition;
+import com.radixdlt.protocol.ProtocolUpdateTrigger;
 import com.radixdlt.sbor.codec.CodecMap;
 import com.radixdlt.sbor.codec.EnumCodec;
 import com.radixdlt.sbor.codec.StructCodec;
@@ -101,11 +101,11 @@ public static void registerCodec(CodecMap codecMap) {
   }
 
   public static ProtocolState testingEmpty() {
-    return new ProtocolState("testing-genesis", ImmutableMap.of(), ImmutableList.of());
+    return new ProtocolState("babylon-genesis", ImmutableMap.of(), ImmutableList.of());
   }
 
   public record PendingProtocolUpdate(
-      ProtocolUpdate protocolUpdate, PendingProtocolUpdateState state) {}
+      ProtocolUpdateTrigger protocolUpdateTrigger, PendingProtocolUpdateState state) {}
 
   public sealed interface PendingProtocolUpdateState {
     record ForSignalledReadinessSupportCondition(

core-rust-bridge/src/test/java/com/radixdlt/rev2/ProtocolConfigGeneratorTest.java

@@ -67,9 +67,10 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.reflect.TypeToken;
 import com.radixdlt.protocol.ProtocolConfig;
-import com.radixdlt.protocol.ProtocolUpdate;
 import com.radixdlt.protocol.ProtocolUpdateEnactmentCondition;
+import com.radixdlt.protocol.ProtocolUpdateTrigger;
 import com.radixdlt.sbor.NodeSborCodecs;
+import java.util.Map;
 import org.bouncycastle.util.encoders.Hex;
 import org.junit.Ignore;
 import org.junit.Test;
@@ -80,14 +81,26 @@ public final class ProtocolConfigGeneratorTest {
   @Test
   public void generateProtocolConfig() {
     // Change this, then run the test to generate the config
+    // NOTE:
+    // - See `resolve_update_definition_for_version` in `protocol_definition_resolver.rs` for the
+    // supported protocol versions
+    // - Any protocol version starting "test-" will add a single transaction to ledger at enactment
+    // - Any protocol version starting "custom-" can be configured with overrides to commit
+    // arbitrary flash transactions. To do this, add an overrides map of protocol version => SBOR
+    // encoded bytes of Vec<Vec<UpdateTransaction>>. Where UpdateTransaction is an enum with one
+    // option at present (FlashTransactionV1). These SBOR encoded bytes should be created in Rust.
+    // See e.g. protocol/test.rs - although we should add an easier-to-use generator if we choose
+    // to use this in tests. Please ask if you need help with this.
     final var protocolConfig =
         new ProtocolConfig(
-            "babylon-genesis",
+            // List of triggers
             ImmutableList.of(
-                new ProtocolUpdate(
-                    "v2",
+                new ProtocolUpdateTrigger(
+                    "test-v2",
                     ProtocolUpdateEnactmentCondition.singleReadinessThresholdBetweenEpochs(
-                        5, 20, Decimal.ofNonNegativeFraction(7, 10), 1))));
+                        5, 20, Decimal.ofNonNegativeFraction(7, 10), 1))),
+            // Overrides map
+            Map.of());
 
     final var protocolConfigBytes =
         NodeSborCodecs.encode(protocolConfig, NodeSborCodecs.resolveCodec(new TypeToken<>() {}));

core-rust/Cargo.toml

@@ -5,6 +5,7 @@ members = [
     "core-api-server",
     "state-manager",
 ]
+resolver = "2"
 
 [workspace.dependencies]
 # DEPENDENCIES ON RADIXDLT-SCRYPTO

core-rust/core-api-server/src/core_api/handlers/status_network_status.rs

@@ -78,7 +78,7 @@ pub(crate) async fn handle_status_network_status(
                 )?))
             })
             .transpose()?,
-        current_protocol_version,
+        current_protocol_version: current_protocol_version.to_string(),
     }))
 }
 

core-rust/state-manager/src/jni/node_rust_environment.rs

@@ -87,11 +87,7 @@ use crate::store::StateManagerDatabase;
 
 use super::fatal_panic_handler::FatalPanicHandler;
 
-use crate::mainnet_updates::ProductionProtocolUpdaterFactory;
-use crate::{
-    ProtocolUpdaterFactory, StateComputer, StateManager, StateManagerConfig,
-    TestingDefaultProtocolUpdaterFactory,
-};
+use crate::{StateComputer, StateManager, StateManagerConfig};
 
 const POINTER_JNI_FIELD_NAME: &str = "rustNodeRustEnvironmentPointer";
 
@@ -155,18 +151,10 @@ impl JNINodeRustEnvironment {
             .track_running_tasks()
             .measured(metric_registry.deref());
 
-        let protocol_updater_factory: Box<dyn ProtocolUpdaterFactory + Send + Sync> =
-            if network.id == NetworkDefinition::mainnet().id {
-                Box::new(ProductionProtocolUpdaterFactory::new(network.clone()))
-            } else {
-                Box::new(TestingDefaultProtocolUpdaterFactory::new(network.clone()))
-            };
-
         let state_manager = StateManager::new(
             config,
             Some(MempoolRelayDispatcher::new(env, j_node_rust_env).unwrap()),
             &lock_factory,
-            protocol_updater_factory,
             &metric_registry,
             &scheduler,
         );

core-rust/state-manager/src/jni/protocol_update.rs

@@ -62,7 +62,7 @@
  * permissions under this License.
  */
 
-use crate::{ProtocolConfig, ProtocolUpdate, ProtocolUpdateResult};
+use crate::{protocol::*, ProtocolUpdateResult};
 use jni::objects::{JClass, JObject};
 use jni::sys::jbyteArray;
 use jni::JNIEnv;
@@ -86,7 +86,7 @@ extern "system" fn Java_com_radixdlt_protocol_RustProtocolUpdate_applyProtocolUp
     jni_sbor_coded_fallible_call(
         &env,
         request_payload,
-        |protocol_version_name: String| -> JavaResult<ProtocolUpdateResult> {
+        |protocol_version_name: ProtocolVersionName| -> JavaResult<ProtocolUpdateResult> {
             let result = JNINodeRustEnvironment::get(&env, j_node_rust_env)
                 .state_manager
                 .apply_protocol_update(&protocol_version_name);
@@ -101,18 +101,24 @@ extern "system" fn Java_com_radixdlt_protocol_RustProtocolUpdate_nativeReadiness
     _class: JClass,
     request_payload: jbyteArray,
 ) -> jbyteArray {
-    jni_sbor_coded_call(&env, request_payload, |protocol_update: ProtocolUpdate| {
-        protocol_update.readiness_signal_name()
-    })
+    jni_sbor_coded_call(
+        &env,
+        request_payload,
+        |protocol_update_trigger: ProtocolUpdateTrigger| {
+            protocol_update_trigger.readiness_signal_name()
+        },
+    )
 }
 
 #[no_mangle]
-extern "system" fn Java_com_radixdlt_protocol_RustProtocolUpdate_nativeMainnetConfig(
+extern "system" fn Java_com_radixdlt_protocol_RustProtocolUpdate_nativeResolveForNetwork(
     env: JNIEnv,
     _class: JClass,
     request_payload: jbyteArray,
 ) -> jbyteArray {
-    jni_sbor_coded_call(&env, request_payload, |_: ()| ProtocolConfig::mainnet())
+    jni_sbor_coded_call(&env, request_payload, |network: NetworkDefinition| {
+        resolve_protocol_config(&network)
+    })
 }
 
 pub fn export_extern_functions() {}

core-rust/state-manager/src/jni/state_computer.rs

@@ -62,7 +62,8 @@
  * permissions under this License.
  */
 
-use crate::{CommitSummary, LedgerProof, ProtocolState};
+use crate::protocol::ProtocolVersionName;
+use crate::{protocol::ProtocolState, CommitSummary, LedgerProof};
 use jni::objects::{JClass, JObject};
 use jni::sys::jbyteArray;
 use jni::JNIEnv;
@@ -191,7 +192,7 @@ extern "system" fn Java_com_radixdlt_statecomputer_RustStateComputer_newestProto
     j_node_rust_env: JObject,
     request_payload: jbyteArray,
 ) -> jbyteArray {
-    jni_sbor_coded_call(&env, request_payload, |_: ()| -> String {
+    jni_sbor_coded_call(&env, request_payload, |_: ()| -> ProtocolVersionName {
         let env = JNINodeRustEnvironment::get(&env, j_node_rust_env);
         env.state_manager.newest_protocol_version()
     })

core-rust/state-manager/src/jni/transaction_store.rs

@@ -64,8 +64,9 @@
 
 use crate::jni::node_rust_environment::JNINodeRustEnvironment;
 use crate::jni::LedgerSyncLimitsConfig;
+use crate::protocol::epoch_change_iter;
 use crate::store::traits::*;
-use crate::{epoch_change_iter, LedgerProof, StateVersion};
+use crate::{LedgerProof, StateVersion};
 use jni::objects::{JClass, JObject};
 use jni::sys::jbyteArray;
 use jni::JNIEnv;

core-rust/state-manager/src/lib.rs

@@ -85,7 +85,6 @@ mod test;
 pub use crate::mempool::*;
 pub use crate::metrics::*;
 pub use crate::pending_transaction_result_cache::*;
-pub use crate::protocol::*;
 pub use crate::receipt::*;
 pub use crate::staging::*;
 pub use crate::state_computer::*;