schemas: PoV compatible changes

common/primitives/src/schema.rs

@@ -73,10 +73,27 @@ pub struct SchemaResponse {
 	pub settings: Vec<SchemaSetting>,
 }
 
+/// RPC Response form for a Schema Info
+#[cfg_attr(feature = "std", derive(Serialize, Deserialize))]
+#[derive(Clone, Encode, Decode, PartialEq, Debug, TypeInfo, Eq)]
+pub struct SchemaInfoResponse {
+	/// The unique identifier for this Schema
+	pub schema_id: SchemaId,
+	/// The model format type for how the schema model is represented
+	pub model_type: ModelType,
+	/// The payload location
+	pub payload_location: PayloadLocation,
+	/// grants for the schema
+	pub settings: Vec<SchemaSetting>,
+}
+
 /// This allows other pallets to resolve Schema information. With generic SchemaId
 pub trait SchemaProvider<SchemaId> {
 	/// Gets the Schema details associated with this `SchemaId` if any
 	fn get_schema_by_id(schema_id: SchemaId) -> Option<SchemaResponse>;
+
+	/// Gets the Schema Info associated with this `SchemaId` if any
+	fn get_schema_info_by_id(schema_id: SchemaId) -> Option<SchemaInfoResponse>;
 }
 
 /// This allows other Pallets to check validity of schema ids.

designdocs/schema_v2.md

@@ -0,0 +1,41 @@
+# On Chain Message Storage
+
+## Context and Scope
+The proposed feature consists of changes that is going to be one (or more) pallet(s) in runtime of a
+Substrate based blockchain, and it will be used in all environments including production.
+
+## Problem Statement
+After introduction of **Proof of Validity** or **PoV** in runtime weights, all pallets should be
+re-evaluated and refactored if necessary to minimize the usage of **PoV**. This is to ensure all
+important operations are scalable.
+This document tries to propose some changes on **Schemas** pallet to optimize the **PoV** size.
+
+## Goals
+- Minimizing Weights including **execution times** and **PoV** size.
+
+## Proposal
+Split Schemas into `SchemaInfo` and `payload` would allow lower **PoV** when verifying schema existence
+or compatibility.
+
+### Main Storage types
+- **SchemaInfos**
+    - _Type_: `StorageMap<SchemaId, SchemaInfo>`
+    - _Purpose_: Main structure To store related properties of any schema
+      index
+- **SchemaPayloads**
+    -  _Type_: `StorageMap<SchemaId, BoundedVec<u8>>`
+    - _Purpose_: Stores the payload or model for each schema
+
+
+### On Chain Structure
+Following is a proposed data structure for storing schema information on chain.
+```rust
+pub struct SchemaInfo {
+    /// The type of model (AvroBinary, Parquet, etc.)
+    pub model_type: ModelType,
+    /// The payload location
+    pub payload_location: PayloadLocation,
+    /// additional control settings for the schema
+    pub settings: SchemaSettings,
+}
+```

node/cli/Cargo.toml

@@ -29,6 +29,7 @@ cli-opt = { default-features = false, path = "../cli-opt" }
 
 # Substrate
 frame-benchmarking-cli = { git = "https://github.com/paritytech/polkadot-sdk", optional = true, branch = "release-polkadot-v1.1.0" }
+frame-benchmarking = { git = "https://github.com/paritytech/polkadot-sdk", optional = true, branch = "release-polkadot-v1.1.0" }
 frame-support = { git = "https://github.com/paritytech/polkadot-sdk", default-features = false, branch = "release-polkadot-v1.1.0" }
 frame-system = { git = "https://github.com/paritytech/polkadot-sdk", default-features = false, branch = "release-polkadot-v1.1.0" }
 pallet-balances = { git = "https://github.com/paritytech/polkadot-sdk", default-features = false, branch = "release-polkadot-v1.1.0" }
@@ -70,6 +71,7 @@ cli = [
   "sc-cli",
   "sc-service",
   "frame-benchmarking-cli",
+  "frame-benchmarking",
   "try-runtime-cli"
 ]
 default = ["std", "cli"]

node/cli/src/command.rs

@@ -371,22 +371,22 @@ pub fn run() -> Result<()> {
 
 		#[cfg(feature = "try-runtime")]
 		Some(Subcommand::TryRuntime(cmd)) => {
-			use sc_executor::{sp_wasm_interface::ExtendedHostFunctions, NativeExecutionDispatch};
+			use common_runtime::constants::MILLISECS_PER_BLOCK;
+			use try_runtime_cli::block_building_info::timestamp_with_aura_info;
 			let runner = cli.create_runner(cmd)?;
-			runner.async_run(|config| {
-				// we don't need any of the components of new_partial, just a runtime, or a task
-				// manager to do `async_run`.
-				let registry = config.prometheus_config.as_ref().map(|cfg| &cfg.registry);
-				let task_manager =
-					sc_service::TaskManager::new(config.tokio_handle.clone(), registry)
-						.map_err(|e| sc_cli::Error::Service(sc_service::Error::Prometheus(e)))?;
-				Ok((
-					cmd.run::<Block, ExtendedHostFunctions<
-						sp_io::SubstrateHostFunctions,
-						<ExecutorDispatch as NativeExecutionDispatch>::ExtendHostFunctions,
-					>>(),
-					task_manager,
-				))
+			type HostFunctions =
+				(sp_io::SubstrateHostFunctions, frame_benchmarking::benchmarking::HostFunctions);
+
+			// grab the task manager.
+			let registry = &runner.config().prometheus_config.as_ref().map(|cfg| &cfg.registry);
+			let task_manager =
+				sc_service::TaskManager::new(runner.config().tokio_handle.clone(), *registry)
+					.map_err(|e| format!("Error: {:?}", e))?;
+
+			let info_provider = timestamp_with_aura_info(MILLISECS_PER_BLOCK);
+
+			runner.async_run(|_| {
+				Ok((cmd.run::<Block, HostFunctions, _>(Some(info_provider)), task_manager))
 			})
 		},
 		Some(Subcommand::ExportRuntimeVersion(cmd)) => {

pallets/messages/src/lib.rs

@@ -230,7 +230,7 @@ pub mod pallet {
 				.try_into()
 				.map_err(|_| Error::<T>::ExceedsMaxMessagePayloadSizeBytes)?;
 
-			if let Some(schema) = T::SchemaProvider::get_schema_by_id(schema_id) {
+			if let Some(schema) = T::SchemaProvider::get_schema_info_by_id(schema_id) {
 				ensure!(
 					schema.payload_location == PayloadLocation::IPFS,
 					Error::<T>::InvalidPayloadLocation
@@ -283,7 +283,7 @@ pub mod pallet {
 			let bounded_payload: BoundedVec<u8, T::MessagesMaxPayloadSizeBytes> =
 				payload.try_into().map_err(|_| Error::<T>::ExceedsMaxMessagePayloadSizeBytes)?;
 
-			if let Some(schema) = T::SchemaProvider::get_schema_by_id(schema_id) {
+			if let Some(schema) = T::SchemaProvider::get_schema_info_by_id(schema_id) {
 				ensure!(
 					schema.payload_location == PayloadLocation::OnChain,
 					Error::<T>::InvalidPayloadLocation

pallets/messages/src/tests/mock.rs

@@ -213,6 +213,17 @@ impl SchemaProvider<u16> for SchemaHandler {
 			settings: Vec::new(),
 		})
 	}
+
+	fn get_schema_info_by_id(schema_id: u16) -> Option<SchemaInfoResponse> {
+		Self::get_schema_by_id(schema_id).and_then(|schema| {
+			Some(SchemaInfoResponse {
+				schema_id: schema.schema_id,
+				settings: schema.settings,
+				model_type: schema.model_type,
+				payload_location: schema.payload_location,
+			})
+		})
+	}
 }
 
 impl pallet_messages::Config for Test {

pallets/messages/src/weights.rs

@@ -18,9 +18,9 @@
 //! Autogenerated weights for pallet_messages
 //!
 //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev
-//! DATE: 2023-10-30, STEPS: `20`, REPEAT: `10`, LOW RANGE: `[]`, HIGH RANGE: `[]`
+//! DATE: 2023-11-02, STEPS: `20`, REPEAT: `10`, LOW RANGE: `[]`, HIGH RANGE: `[]`
 //! WORST CASE MAP SIZE: `1000000`
-//! HOSTNAME: `benchmark-runner-44wtw-bw25f`, CPU: `Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz`
+//! HOSTNAME: `benchmark-runner-44wtw-n2ql6`, CPU: `Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz`
 //! EXECUTION: , WASM-EXECUTION: Compiled, CHAIN: Some("frequency-bench"), DB CACHE: 1024
 
 // Executed Command:
@@ -56,8 +56,8 @@ pub trait WeightInfo {
 /// Weights for pallet_messages using the Substrate node and recommended hardware.
 pub struct SubstrateWeight<T>(PhantomData<T>);
 impl<T: frame_system::Config> WeightInfo for SubstrateWeight<T> {
-	/// Storage: `Schemas::Schemas` (r:1 w:0)
-	/// Proof: `Schemas::Schemas` (`max_values`: None, `max_size`: Some(65518), added: 67993, mode: `Measured`)
+	/// Storage: `Schemas::SchemaInfos` (r:1 w:0)
+	/// Proof: `Schemas::SchemaInfos` (`max_values`: None, `max_size`: Some(14), added: 2489, mode: `Measured`)
 	/// Storage: `Msa::PublicKeyToMsaId` (r:1 w:0)
 	/// Proof: `Msa::PublicKeyToMsaId` (`max_values`: None, `max_size`: Some(48), added: 2523, mode: `Measured`)
 	/// Storage: `Msa::DelegatorAndProviderToDelegation` (r:1 w:0)
@@ -67,36 +67,36 @@ impl<T: frame_system::Config> WeightInfo for SubstrateWeight<T> {
 	/// The range of component `n` is `[0, 3071]`.
 	fn add_onchain_message(n: u32, ) -> Weight {
 		// Proof Size summary in bytes:
-		//  Measured:  `9664`
-		//  Estimated: `22039`
-		// Minimum execution time: 47_640_000 picoseconds.
-		Weight::from_parts(49_189_621, 22039)
-			// Standard Error: 82
-			.saturating_add(Weight::from_parts(938, 0).saturating_mul(n.into()))
+		//  Measured:  `9623`
+		//  Estimated: `21998`
+		// Minimum execution time: 46_448_000 picoseconds.
+		Weight::from_parts(47_830_962, 21998)
+			// Standard Error: 67
+			.saturating_add(Weight::from_parts(957, 0).saturating_mul(n.into()))
 			.saturating_add(T::DbWeight::get().reads(4_u64))
 			.saturating_add(T::DbWeight::get().writes(1_u64))
 	}
-	/// Storage: `Schemas::Schemas` (r:1 w:0)
-	/// Proof: `Schemas::Schemas` (`max_values`: None, `max_size`: Some(65518), added: 67993, mode: `Measured`)
+	/// Storage: `Schemas::SchemaInfos` (r:1 w:0)
+	/// Proof: `Schemas::SchemaInfos` (`max_values`: None, `max_size`: Some(14), added: 2489, mode: `Measured`)
 	/// Storage: `Msa::PublicKeyToMsaId` (r:1 w:0)
 	/// Proof: `Msa::PublicKeyToMsaId` (`max_values`: None, `max_size`: Some(48), added: 2523, mode: `Measured`)
 	/// Storage: `Messages::Messages` (r:1 w:1)
 	/// Proof: `Messages::Messages` (`max_values`: None, `max_size`: Some(618624), added: 621099, mode: `Measured`)
 	fn add_ipfs_message() -> Weight {
 		// Proof Size summary in bytes:
-		//  Measured:  `7958`
-		//  Estimated: `20333`
-		// Minimum execution time: 43_681_000 picoseconds.
-		Weight::from_parts(45_218_000, 20333)
+		//  Measured:  `7833`
+		//  Estimated: `20208`
+		// Minimum execution time: 45_816_000 picoseconds.
+		Weight::from_parts(47_622_000, 20208)
 			.saturating_add(T::DbWeight::get().reads(3_u64))
 			.saturating_add(T::DbWeight::get().writes(1_u64))
 	}
 }
 
 // For backwards compatibility and tests
 impl WeightInfo for () {
-	/// Storage: `Schemas::Schemas` (r:1 w:0)
-	/// Proof: `Schemas::Schemas` (`max_values`: None, `max_size`: Some(65518), added: 67993, mode: `Measured`)
+	/// Storage: `Schemas::SchemaInfos` (r:1 w:0)
+	/// Proof: `Schemas::SchemaInfos` (`max_values`: None, `max_size`: Some(14), added: 2489, mode: `Measured`)
 	/// Storage: `Msa::PublicKeyToMsaId` (r:1 w:0)
 	/// Proof: `Msa::PublicKeyToMsaId` (`max_values`: None, `max_size`: Some(48), added: 2523, mode: `Measured`)
 	/// Storage: `Msa::DelegatorAndProviderToDelegation` (r:1 w:0)
@@ -106,27 +106,27 @@ impl WeightInfo for () {
 	/// The range of component `n` is `[0, 3071]`.
 	fn add_onchain_message(n: u32, ) -> Weight {
 		// Proof Size summary in bytes:
-		//  Measured:  `9664`
-		//  Estimated: `22039`
-		// Minimum execution time: 47_640_000 picoseconds.
-		Weight::from_parts(49_189_621, 22039)
-			// Standard Error: 82
-			.saturating_add(Weight::from_parts(938, 0).saturating_mul(n.into()))
+		//  Measured:  `9623`
+		//  Estimated: `21998`
+		// Minimum execution time: 46_448_000 picoseconds.
+		Weight::from_parts(47_830_962, 21998)
+			// Standard Error: 67
+			.saturating_add(Weight::from_parts(957, 0).saturating_mul(n.into()))
 			.saturating_add(RocksDbWeight::get().reads(4_u64))
 			.saturating_add(RocksDbWeight::get().writes(1_u64))
 	}
-	/// Storage: `Schemas::Schemas` (r:1 w:0)
-	/// Proof: `Schemas::Schemas` (`max_values`: None, `max_size`: Some(65518), added: 67993, mode: `Measured`)
+	/// Storage: `Schemas::SchemaInfos` (r:1 w:0)
+	/// Proof: `Schemas::SchemaInfos` (`max_values`: None, `max_size`: Some(14), added: 2489, mode: `Measured`)
 	/// Storage: `Msa::PublicKeyToMsaId` (r:1 w:0)
 	/// Proof: `Msa::PublicKeyToMsaId` (`max_values`: None, `max_size`: Some(48), added: 2523, mode: `Measured`)
 	/// Storage: `Messages::Messages` (r:1 w:1)
 	/// Proof: `Messages::Messages` (`max_values`: None, `max_size`: Some(618624), added: 621099, mode: `Measured`)
 	fn add_ipfs_message() -> Weight {
 		// Proof Size summary in bytes:
-		//  Measured:  `7958`
-		//  Estimated: `20333`
-		// Minimum execution time: 43_681_000 picoseconds.
-		Weight::from_parts(45_218_000, 20333)
+		//  Measured:  `7833`
+		//  Estimated: `20208`
+		// Minimum execution time: 45_816_000 picoseconds.
+		Weight::from_parts(47_622_000, 20208)
 			.saturating_add(RocksDbWeight::get().reads(3_u64))
 			.saturating_add(RocksDbWeight::get().writes(1_u64))
 	}

pallets/schemas/Cargo.toml

@@ -17,6 +17,7 @@ targets = ["x86_64-unknown-linux-gnu"]
 codec = { package = "parity-scale-codec", version = "3.6.1", default-features = false, features = [
   "derive",
 ] }
+log = { version = "0.4.17", default-features = false }
 frame-benchmarking = { default-features = false, git = "https://github.com/paritytech/polkadot-sdk", optional = true, branch = "release-polkadot-v1.1.0" }
 frame-support = { default-features = false, git = "https://github.com/paritytech/polkadot-sdk", branch = "release-polkadot-v1.1.0" }
 frame-system = { default-features = false, git = "https://github.com/paritytech/polkadot-sdk", branch = "release-polkadot-v1.1.0" }

pallets/schemas/src/benchmarking.rs

@@ -42,7 +42,7 @@ benchmarks! {
 	}: _(RawOrigin::Signed(sender), schema_input, model_type, payload_location)
 	verify {
 		ensure!(SchemasPallet::<T>::get_current_schema_identifier_maximum() > 0, "Created schema count should be > 0");
-		ensure!(SchemasPallet::<T>::get_schema(1).is_some(), "Created schema should exist");
+		ensure!(SchemasPallet::<T>::get_schema_info(1).is_some(), "Created schema should exist");
 	}
 
 	create_schema_via_governance {
@@ -55,7 +55,7 @@ benchmarks! {
 	}: _(RawOrigin::Root, sender.clone(), schema_input, model_type, payload_location, BoundedVec::default())
 	verify {
 		ensure!(SchemasPallet::<T>::get_current_schema_identifier_maximum() > 0, "Created schema count should be > 0");
-		ensure!(SchemasPallet::<T>::get_schema(1).is_some(), "Created schema should exist");
+		ensure!(SchemasPallet::<T>::get_schema_info(1).is_some(), "Created schema should exist");
 	}
 
 	propose_to_create_schema {
@@ -80,7 +80,7 @@ benchmarks! {
 	}: _(RawOrigin::Signed(sender), schema_input, model_type, payload_location, BoundedVec::default())
 	verify {
 		ensure!(SchemasPallet::<T>::get_current_schema_identifier_maximum() > 0, "Created schema count should be > 0");
-		ensure!(SchemasPallet::<T>::get_schema(1).is_some(), "Created schema should exist");
+		ensure!(SchemasPallet::<T>::get_schema_info(1).is_some(), "Created schema should exist");
 	}
 
 	set_max_schema_model_bytes {