enhancement: store raw data for unknown transaction kinds in order to capture them so they can be parsed later (#183)

* enhancement: store raw data for unknown transaction kinds in order to capture them so they can be parsed later

* [checksums] Add all tx_* wasms from namada-sdk

* [transactions] Add on_conflict handling so that old blocks can be re-crawled for transactions without failing

Author: joel-u410

orm/src/transactions.rs

@@ -58,7 +58,7 @@ impl From<TransactionKind> for TransactionKindDb {
             TransactionKind::BecomeValidator(_) => {
                 TransactionKindDb::BecomeValidator
             }
-            TransactionKind::Unknown => TransactionKindDb::Unknown,
+            TransactionKind::Unknown(_) => TransactionKindDb::Unknown,
         }
     }
 }

parameters/src/repository/parameters.rs

@@ -19,6 +19,8 @@ pub fn upsert_chain_parameters(
                 .eq(excluded(chain_parameters::max_block_time)),
             chain_parameters::cubic_slashing_window_length
                 .eq(excluded(chain_parameters::cubic_slashing_window_length)),
+            chain_parameters::checksums
+                .eq(excluded(chain_parameters::checksums)),
         ))
         .execute(transaction_conn)
         .context("Failed to update chain_parameters state in db")?;

shared/src/checksums.rs

@@ -1,9 +1,13 @@
 use bimap::BiMap;
 use namada_sdk::tx::{
-    TX_BECOME_VALIDATOR_WASM, TX_BOND_WASM, TX_CHANGE_COMMISSION_WASM,
-    TX_CHANGE_METADATA_WASM, TX_CLAIM_REWARDS_WASM, TX_IBC_WASM,
-    TX_INIT_PROPOSAL, TX_REDELEGATE_WASM, TX_REVEAL_PK, TX_TRANSFER_WASM,
-    TX_UNBOND_WASM, TX_VOTE_PROPOSAL, TX_WITHDRAW_WASM,
+    TX_BECOME_VALIDATOR_WASM, TX_BOND_WASM, TX_BRIDGE_POOL_WASM,
+    TX_CHANGE_COMMISSION_WASM, TX_CHANGE_CONSENSUS_KEY_WASM,
+    TX_CHANGE_METADATA_WASM, TX_CLAIM_REWARDS_WASM,
+    TX_DEACTIVATE_VALIDATOR_WASM, TX_IBC_WASM, TX_INIT_ACCOUNT_WASM,
+    TX_INIT_PROPOSAL, TX_REACTIVATE_VALIDATOR_WASM, TX_REDELEGATE_WASM,
+    TX_RESIGN_STEWARD, TX_REVEAL_PK, TX_TRANSFER_WASM, TX_UNBOND_WASM,
+    TX_UNJAIL_VALIDATOR_WASM, TX_UPDATE_ACCOUNT_WASM,
+    TX_UPDATE_STEWARD_COMMISSION, TX_VOTE_PROPOSAL, TX_WITHDRAW_WASM,
 };
 use serde::{Deserialize, Serialize};
 
@@ -44,6 +48,15 @@ impl Checksums {
             TX_CHANGE_COMMISSION_WASM.to_string(),
             TX_IBC_WASM.to_string(),
             TX_BECOME_VALIDATOR_WASM.to_string(),
+            TX_INIT_ACCOUNT_WASM.to_string(),
+            TX_UNJAIL_VALIDATOR_WASM.to_string(),
+            TX_DEACTIVATE_VALIDATOR_WASM.to_string(),
+            TX_REACTIVATE_VALIDATOR_WASM.to_string(),
+            TX_UPDATE_ACCOUNT_WASM.to_string(),
+            TX_BRIDGE_POOL_WASM.to_string(),
+            TX_CHANGE_CONSENSUS_KEY_WASM.to_string(),
+            TX_RESIGN_STEWARD.to_string(),
+            TX_UPDATE_STEWARD_COMMISSION.to_string(),
         ]
     }
 }

shared/src/transaction.rs

@@ -29,6 +29,39 @@ pub struct RevealPkData {
     pub public_key: PublicKey,
 }
 
+// Capture details for unknown transactions so we can store them in the db
+#[derive(Serialize, Debug, Clone)]
+pub struct UnknownTransaction {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub id: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub name: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(serialize_with = "serialize_optional_bytes_to_hex")]
+    pub data: Option<Vec<u8>>,
+}
+
+fn serialize_optional_bytes_to_hex<S>(
+    bytes: &Option<Vec<u8>>,
+    serializer: S,
+) -> Result<S::Ok, S::Error>
+where
+    S: serde::Serializer,
+{
+    bytes
+        .as_ref()
+        .map(|b| {
+            let mut s = String::with_capacity(2 + (b.len() * 2)); // "0x" + 2 chars per byte
+            s.push_str("0x");
+            for byte in b {
+                use std::fmt::Write;
+                write!(s, "{:02x}", byte).unwrap();
+            }
+            s
+        })
+        .serialize(serializer)
+}
+
 #[derive(Serialize, Debug, Clone)]
 #[serde(untagged)]
 pub enum TransactionKind {
@@ -48,15 +81,15 @@ pub enum TransactionKind {
     CommissionChange(Option<CommissionChange>),
     RevealPk(Option<RevealPkData>),
     BecomeValidator(Option<Box<BecomeValidator>>),
-    Unknown,
+    Unknown(Option<UnknownTransaction>),
 }
 
 impl TransactionKind {
     pub fn to_json(&self) -> Option<String> {
         serde_json::to_string(&self).ok()
     }
 
-    pub fn from(tx_kind_name: &str, data: &[u8]) -> Self {
+    pub fn from(id: &str, tx_kind_name: &str, data: &[u8]) -> Self {
         match tx_kind_name {
             "tx_transfer" => {
                 let data = if let Ok(data) = Transfer::try_from_slice(data) {
@@ -174,7 +207,11 @@ impl TransactionKind {
             }
             _ => {
                 tracing::warn!("Unknown transaction kind: {}", tx_kind_name);
-                TransactionKind::Unknown
+                TransactionKind::Unknown(Some(UnknownTransaction {
+                    id: Some(id.to_string()),
+                    name: Some(tx_kind_name.to_string()),
+                    data: Some(data.to_vec()),
+                }))
             }
         }
     }
@@ -325,12 +362,20 @@ impl Transaction {
                         if let Some(tx_kind_name) =
                             checksums.get_name_by_id(&id)
                         {
-                            TransactionKind::from(&tx_kind_name, &tx_data)
+                            TransactionKind::from(&id, &tx_kind_name, &tx_data)
                         } else {
-                            TransactionKind::Unknown
+                            TransactionKind::Unknown(Some(UnknownTransaction {
+                                id: Some(id),
+                                name: None,
+                                data: Some(tx_data.clone()),
+                            }))
                         }
                     } else {
-                        TransactionKind::Unknown
+                        TransactionKind::Unknown(Some(UnknownTransaction {
+                            id: None,
+                            name: None,
+                            data: Some(tx_data.clone()),
+                        }))
                     };
 
                     let encoded_tx_data = if !tx_data.is_empty() {

transactions/src/repository/transactions.rs

@@ -18,6 +18,15 @@ pub fn insert_inner_transactions(
                 .map(InnerTransactionInsertDb::from)
                 .collect::<Vec<_>>(),
         )
+        .on_conflict(inner_transactions::id)
+        .do_update()
+        .set((
+            // Allow updating transactions kind + data so that if the indexer is updated with
+            // new transaction type support, we can easily go back & reindex any old transactions
+            // that were previously marked as "unknown".
+            inner_transactions::kind.eq(excluded(inner_transactions::kind)),
+            inner_transactions::data.eq(excluded(inner_transactions::data)),
+        ))
         .execute(transaction_conn)
         .context("Failed to insert inner transactions in db")?;
 
@@ -34,6 +43,7 @@ pub fn insert_wrapper_transactions(
                 .map(WrapperTransactionInsertDb::from)
                 .collect::<Vec<_>>(),
         )
+        .on_conflict_do_nothing()
         .execute(transaction_conn)
         .context("Failed to insert wrapper transactions in db")?;