Port Polkadot implementation to Rust. (#4102)

* Add the tw_ss58_address crate

* Apply suggestions from code review, refactor some utility functions

* Remove unused dependency

* Add PolkadotAddress

* Add NetworkId constants and Hash derivation

* Add SCALE encoding for integers (fixed and compact)

* Fix tiny things in scale.rs

* Add extrinsics

* Add PolkadotPrefix

* Require a pre-built NetworkId when deriving SS58Address

* Format extrinsic.rs

* Add address parsing and derivation in entry.rs

* Rename SS58Address::from_str to SS58Address::parse

* Add SubstrateNetwork to AddressPrefix enum

* Rework ToScale trait to allow encoding to an existing Vec, add more implementations

* Refactor Extrinsic to use in-place encoding

* Add more tests for staking extrinsics

* Introduce ExtrinsicEncoder helper to build encoded extrinsics

* Refactor Scale encoding into tw_scale crate.

* Add u128 SCALE support.

* Fix: Use correct endianess.

* Add struct/enum SCALE encoding macros.

* Add Substrate ExtrinsicV4 encoding.

* U256 to u128.

* Add Substrate call encoding support.

* Fix some incorrect tests.

* Add more extrinsic support.

* Fix Option SCALE encoding.

* Add Polymesh network id.

* Remove extra lifetimes.  Simplify code.

* More encoder refactoring.

* Fix SCALE encoding test for Option type.

* Update Polkadot CoinEntry impl.

* Add support for custom call index.

* Implement Era SCALE encoding.

* cargo fmt

* Move common Substrate logic in tw_substrate crate.

* Improve bool SCALE code.

* Refactor highly nested test code.

* cargo fmt

* Implment Polkadot coin address derivation test.

* Add more Era tests ported from the C++ tests.

* Refactor call encoder.

* FIXUP

* Implement signer and compiler.

* Start adding tw_tests for polkadot.

* Add ss58prefix support to tw_coin_registry.

* implement tw_any_address_is_valid_ss58 and tw_any_address_create_ss58_with_public_key

* Fix missing length prefix when encoding signed transaction.

* cargo fmt

* Add support for CheckMetadata transaction extension.

* Refactor transaction building.

* Add support for optional MultiAddress.

* Finish support for optional MultiAddress.

* Fix C++ test, Rust impl returns different error code.

* Fix 'tip' support.

* Add support for ChargeAssetTxPayment (paying tx fee with asset).

* Remove unused call indices.

* Add more SS58 address types.

* Update Polkadot C++ tests to use TWAny* interfaces.

* Use default value for block hashes.

* Improve errors.

* Support custom call indices in Polymesh.

* Fix Polkadot CheckMetadataHash support

* Support old staking.bond calls.

* Support Kusama with Rust impl.  Remove old C++ impl.

* Fix address derivation test for Acala and Kusama

* Add Rust polkadot compile/sign tests.

* cargo fmt

* Merge TransactionBuilder and PrepareTransaction types.

* Use SubstrateCoinEntry trait to better support Substrate chains.

* Add missing crates to top-level Cargo.toml.

* Re-enable test.

* Fix memory leaks in C++ unit tests.

* Update Acala support to match C++.

* Move tw_substrate to frameworks.

* Improve error reporting.

* Simplify Compact encoding.

* Refactor some rarely used SCALE code.

* Rename Encoded to RawOwned and move to tw_scale.

* Use H256 in Memo and IdentityId.

* Don't store KeyPair.

* Remove unused error variant.

* Improve errors.

* Improve errors with context.

* Remove TWSS58AddressType

* Remove deadcode

* Remove consts from tw_ss58_address crate.

* Simplify impl_enum_scale macro

* Document SCALE macros.

* Some code cleanup and deps reorder.

* Cleanup `parse_address`

* Fix RewardDestination.  Added missing ACCOUNT variant.

* Remove assert.

* Fix C++ tests and use original expected output.

* Fix TODO

* Revert RewardDestination::ACCOUNT change.

* Add links for Polymesh encoded transactions.

* Cleanup error conversion code.

* Port some tests from C++ to Rust.

* Finish porting C++ test to Rust.

* Remove old C++ tests.  Only keep one for integration.

* Sort crates.

* Add docs to `tw_substrate`.

* More docs.

* Add note about space padded Memo.

* Port most address validation tests from C++.

* Fix cargo clippy warnings.

* Fix C++ test compile errors.

* Fix implicit conversion warning.

* Try fixing iOS build.

---------

Co-authored-by: doom <clement.doumergue@binance.com>

rust/Cargo.toml

@@ -1,4 +1,5 @@
 [workspace]
+resolver = "2"
 members = [
     "chains/tw_aptos",
     "chains/tw_binance",
@@ -11,11 +12,13 @@ members = [
     "chains/tw_native_evmos",
     "chains/tw_native_injective",
     "chains/tw_pactus",
+    "chains/tw_polkadot",
     "chains/tw_ronin",
     "chains/tw_solana",
     "chains/tw_sui",
     "chains/tw_thorchain",
     "chains/tw_ton",
+    "frameworks/tw_substrate",
     "frameworks/tw_ton_sdk",
     "frameworks/tw_utxo",
     "tw_any_coin",
@@ -32,6 +35,8 @@ members = [
     "tw_misc",
     "tw_number",
     "tw_proto",
+    "tw_scale",
+    "tw_ss58_address",
     "tw_tests",
     "wallet_core_bin",
     "wallet_core_rs",

rust/chains/tw_polkadot/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "tw_polkadot"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+tw_coin_entry = { path = "../../tw_coin_entry" }
+tw_encoding = { path = "../../tw_encoding" }
+tw_hash = { path = "../../tw_hash" }
+tw_keypair = { path = "../../tw_keypair" }
+tw_memory = { path = "../../tw_memory" }
+tw_number = { path = "../../tw_number" }
+tw_proto = { path = "../../tw_proto" }
+tw_scale = { path = "../../tw_scale" }
+tw_ss58_address = { path = "../../tw_ss58_address" }
+tw_substrate = { path = "../../frameworks/tw_substrate" }

rust/chains/tw_polkadot/src/call_encoder/generic.rs

@@ -0,0 +1,239 @@
+use std::str::FromStr;
+
+use tw_coin_entry::error::prelude::*;
+use tw_number::U256;
+use tw_proto::Polkadot::Proto::{
+    mod_Balance::{AssetTransfer, OneOfmessage_oneof as BalanceVariant, Transfer},
+    mod_Staking::{
+        Bond, BondExtra, Chill, Nominate, OneOfmessage_oneof as StakingVariant, Rebond, Unbond,
+        WithdrawUnbonded,
+    },
+    Balance, RewardDestination as TWRewardDestination, Staking,
+};
+use tw_scale::{impl_enum_scale, Compact, RawOwned, ToScale};
+use tw_ss58_address::SS58Address;
+use tw_substrate::*;
+
+use super::{required_call_index, validate_call_index};
+
+impl_enum_scale!(
+    #[derive(Clone, Debug)]
+    pub enum GenericBalances {
+        TransferAllowDeath {
+            dest: MultiAddress,
+            value: Compact<u128>,
+        } = 0x00,
+        AssetTransfer {
+            id: Compact<u32>,
+            target: MultiAddress,
+            amount: Compact<u128>,
+        } = 0x05,
+    }
+);
+
+impl GenericBalances {
+    fn encode_transfer(ctx: &SubstrateContext, t: &Transfer) -> WithCallIndexResult<Self> {
+        let ci = validate_call_index(&t.call_indices)?;
+        let address =
+            SS58Address::from_str(&t.to_address).map_err(|_| EncodeError::InvalidAddress)?;
+        let value = U256::from_big_endian_slice(&t.value)
+            .map_err(|_| EncodeError::InvalidValue)?
+            .try_into()
+            .map_err(|_| EncodeError::InvalidValue)?;
+
+        Ok(ci.wrap(Self::TransferAllowDeath {
+            dest: ctx.multi_address(address.into()),
+            value: Compact(value),
+        }))
+    }
+
+    fn encode_asset_transfer(
+        ctx: &SubstrateContext,
+        t: &AssetTransfer,
+    ) -> WithCallIndexResult<Self> {
+        let ci = required_call_index(&t.call_indices)?;
+        let address =
+            SS58Address::from_str(&t.to_address).map_err(|_| EncodeError::InvalidAddress)?;
+        let amount = U256::from_big_endian_slice(&t.value)
+            .map_err(|_| EncodeError::InvalidValue)?
+            .try_into()
+            .map_err(|_| EncodeError::InvalidValue)?;
+
+        let asset_id = t.asset_id;
+        if asset_id > 0 {
+            Ok(ci.wrap(Self::AssetTransfer {
+                id: Compact(asset_id),
+                target: ctx.multi_address(address.into()),
+                amount: Compact(amount),
+            }))
+        } else {
+            Ok(ci.wrap(Self::TransferAllowDeath {
+                dest: ctx.multi_address(address.into()),
+                value: Compact(amount),
+            }))
+        }
+    }
+
+    pub fn encode_call(ctx: &SubstrateContext, b: &Balance) -> WithCallIndexResult<Self> {
+        match &b.message_oneof {
+            BalanceVariant::transfer(t) => Self::encode_transfer(ctx, t),
+            BalanceVariant::asset_transfer(t) => Self::encode_asset_transfer(ctx, t),
+            _ => Err(EncodeError::NotSupported)
+                .into_tw()
+                .context("Unsupported batched balance variants here (maybe nested batch calls?)"),
+        }
+    }
+}
+
+impl_enum_scale!(
+    #[derive(Clone, Debug)]
+    pub enum RewardDestination {
+        Staked = 0x00,
+        Stash = 0x01,
+        Controller = 0x02,
+        None = 0x04,
+    }
+);
+
+impl RewardDestination {
+    pub fn from_tw(dest: &TWRewardDestination) -> EncodeResult<Self> {
+        match dest {
+            TWRewardDestination::STAKED => Ok(Self::Staked),
+            TWRewardDestination::STASH => Ok(Self::Stash),
+            TWRewardDestination::CONTROLLER => Ok(Self::Controller),
+        }
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct BondCall {
+    controller: Option<MultiAddress>,
+    value: Compact<u128>,
+    reward: RewardDestination,
+}
+
+impl ToScale for BondCall {
+    fn to_scale_into(&self, out: &mut Vec<u8>) {
+        if let Some(controller) = &self.controller {
+            controller.to_scale_into(out);
+        }
+        self.value.to_scale_into(out);
+        self.reward.to_scale_into(out);
+    }
+}
+
+impl_enum_scale!(
+    #[derive(Clone, Debug)]
+    pub enum GenericStaking {
+        Bond(BondCall) = 0x00,
+        BondExtra { max_additional: Compact<u128> } = 0x01,
+        Unbond { value: Compact<u128> } = 0x02,
+        WithdrawUnbonded { num_slashing_spans: u32 } = 0x03,
+        Nominate { targets: Vec<MultiAddress> } = 0x05,
+        Chill = 0x06,
+        Rebond { value: Compact<u128> } = 0x13,
+    }
+);
+
+impl GenericStaking {
+    fn encode_bond(ctx: &SubstrateContext, b: &Bond) -> WithCallIndexResult<Self> {
+        let ci = validate_call_index(&b.call_indices)?;
+        let controller = SS58Address::from_str(&b.controller)
+            .map(|addr| ctx.multi_address(addr.into()))
+            .ok();
+        let value = U256::from_big_endian_slice(&b.value)
+            .map_err(|_| EncodeError::InvalidValue)?
+            .try_into()
+            .map_err(|_| EncodeError::InvalidValue)?;
+
+        Ok(ci.wrap(Self::Bond(BondCall {
+            controller,
+            value: Compact(value),
+            reward: RewardDestination::from_tw(&b.reward_destination)?,
+        })))
+    }
+
+    fn encode_bond_extra(b: &BondExtra) -> WithCallIndexResult<Self> {
+        let ci = validate_call_index(&b.call_indices)?;
+        let value = U256::from_big_endian_slice(&b.value)
+            .map_err(|_| EncodeError::InvalidValue)?
+            .try_into()
+            .map_err(|_| EncodeError::InvalidValue)?;
+
+        Ok(ci.wrap(Self::BondExtra {
+            max_additional: Compact(value),
+        }))
+    }
+
+    fn encode_chill(c: &Chill) -> WithCallIndexResult<Self> {
+        let ci = validate_call_index(&c.call_indices)?;
+        Ok(ci.wrap(Self::Chill))
+    }
+
+    fn encode_unbond(b: &Unbond) -> WithCallIndexResult<Self> {
+        let ci = validate_call_index(&b.call_indices)?;
+        let value = U256::from_big_endian_slice(&b.value)
+            .map_err(|_| EncodeError::InvalidValue)?
+            .try_into()
+            .map_err(|_| EncodeError::InvalidValue)?;
+
+        Ok(ci.wrap(Self::Unbond {
+            value: Compact(value),
+        }))
+    }
+
+    fn encode_rebond(b: &Rebond) -> WithCallIndexResult<Self> {
+        let ci = validate_call_index(&b.call_indices)?;
+        let value = U256::from_big_endian_slice(&b.value)
+            .map_err(|_| EncodeError::InvalidValue)?
+            .try_into()
+            .map_err(|_| EncodeError::InvalidValue)?;
+
+        Ok(ci.wrap(Self::Rebond {
+            value: Compact(value),
+        }))
+    }
+
+    fn encode_withdraw_unbonded(b: &WithdrawUnbonded) -> WithCallIndexResult<Self> {
+        let ci = validate_call_index(&b.call_indices)?;
+        Ok(ci.wrap(Self::WithdrawUnbonded {
+            num_slashing_spans: b.slashing_spans as u32,
+        }))
+    }
+
+    fn encode_nominate(ctx: &SubstrateContext, b: &Nominate) -> WithCallIndexResult<Self> {
+        let ci = validate_call_index(&b.call_indices)?;
+        let targets = b
+            .nominators
+            .iter()
+            .map(|target| {
+                let account =
+                    SS58Address::from_str(target).map_err(|_| EncodeError::InvalidAddress)?;
+                Ok(ctx.multi_address(account.into()))
+            })
+            .collect::<EncodeResult<Vec<MultiAddress>>>()?;
+        Ok(ci.wrap(Self::Nominate { targets }))
+    }
+
+    pub fn encode_call(ctx: &SubstrateContext, s: &Staking) -> WithCallIndexResult<Self> {
+        match &s.message_oneof {
+            StakingVariant::bond(b) => Self::encode_bond(ctx, b),
+            StakingVariant::bond_extra(b) => Self::encode_bond_extra(b),
+            StakingVariant::chill(b) => Self::encode_chill(b),
+            StakingVariant::unbond(b) => Self::encode_unbond(b),
+            StakingVariant::withdraw_unbonded(b) => Self::encode_withdraw_unbonded(b),
+            StakingVariant::rebond(b) => Self::encode_rebond(b),
+            StakingVariant::nominate(b) => Self::encode_nominate(ctx, b),
+            _ => Err(EncodeError::NotSupported)
+                .into_tw()
+                .context("Unsupported batched staking variants here (maybe nested batch calls?)"),
+        }
+    }
+}
+
+impl_enum_scale!(
+    #[derive(Clone, Debug)]
+    pub enum GenericUtility {
+        BatchAll { calls: Vec<RawOwned> } = 0x02,
+    }
+);