feat: reconstructing state using CommitBlockInfoV2

Cargo.toml

@@ -14,10 +14,12 @@ blake2 = "0.10.6"
 bytes = "1.5"
 chrono = "0.4.31"
 clap = { version = "4.4.7", features = ["derive", "env"] }
+deflate = { version = "1.0.0", features = ["gzip"] }
 ethers = "1.0.2"
 eyre = "0.6.8"
 hex = "0.4.3"
 indexmap = { version = "2.0.2" }
+primitive-types = "0.12.2"
 prost = "0.12"
 rocksdb = "0.21"
 serde = { version = "1.0.189", features = ["derive"] }
@@ -27,9 +29,8 @@ thiserror = "1.0.50"
 tokio = { version = "1.33.0", features = ["macros"] }
 tracing = "0.1.40"
 tracing-subscriber = "0.3.17"
-zksync_merkle_tree = { git = "https://github.com/matter-labs/zksync-era.git" }
 zkevm_opcode_defs = { git = "https://github.com/matter-labs/era-zkevm_opcode_defs.git" }
-deflate = { version = "1.0.0", features = ["gzip"] }
+zksync_merkle_tree = { git = "https://github.com/matter-labs/zksync-era.git" }
 
 [build-dependencies]
 prost-build = "0.12"

src/main.rs

@@ -41,7 +41,8 @@ fn start_logger(default_level: LevelFilter) {
         _ => EnvFilter::default()
             .add_directive(default_level.into())
             .add_directive("hyper=off".parse().unwrap())
-            .add_directive("ethers=off".parse().unwrap()),
+            .add_directive("ethers=off".parse().unwrap())
+            .add_directive("zksync_storage=off".parse().unwrap()),
     };
 
     tracing_subscriber::fmt()

src/processor/snapshot/mod.rs

@@ -18,7 +18,7 @@ use eyre::Result;
 use prost::Message;
 use state_reconstruct_fetcher::{
     constants::{ethereum, storage},
-    types::CommitBlock,
+    types::{v2::PackingType, CommitBlock},
 };
 use tokio::sync::mpsc;
 
@@ -71,7 +71,20 @@ impl Processor for SnapshotBuilder {
                 self.database
                     .insert_storage_log(&mut SnapshotStorageLog {
                         key: U256::from_little_endian(key),
-                        value: H256::from(value),
+                        value: {
+                            // TODO: procure value depending on packing type. requires reading from
+                            // existing database.
+                            let processed_value = match value {
+                                PackingType::Add(v)
+                                | PackingType::Sub(v)
+                                | PackingType::Transform(v)
+                                | PackingType::NoCompression(v) => v,
+                            };
+
+                            let mut buffer = [0; 32];
+                            processed_value.to_big_endian(&mut buffer);
+                            H256::from(buffer)
+                        },
                         miniblock_number_of_initial_write: U64::from(0),
                         l1_batch_number_of_initial_write: U64::from(
                             block.l1_block_number.unwrap_or(0),
@@ -84,9 +97,24 @@ impl Processor for SnapshotBuilder {
             // Repeated calldata.
             for (index, value) in &block.repeated_storage_changes {
                 let index = usize::try_from(*index).expect("truncation failed");
+                let value = {
+                    // TODO: procure value depending on packing type. requires reading from
+                    // existing database.
+                    let processed_value = match value {
+                        PackingType::Add(v)
+                        | PackingType::Sub(v)
+                        | PackingType::Transform(v)
+                        | PackingType::NoCompression(v) => v,
+                    };
+
+                    let mut buffer = [0; 32];
+                    processed_value.to_big_endian(&mut buffer);
+                    buffer
+                };
+
                 if self
                     .database
-                    .update_storage_log_value(index as u64, value)
+                    .update_storage_log_value(index as u64, &value)
                     .is_err()
                 {
                     let max_idx = self
@@ -190,16 +218,6 @@ fn reconstruct_genesis_state(database: &mut SnapshotDB, path: &str) -> Result<()
 
     for (address, key, value, miniblock_number) in batched {
         let derived_key = derive_final_address_for_params(&address, &key);
-        // TODO: what to do here?
-        // let version = tree.latest_version().unwrap_or_default();
-        // let _leaf = tree.read_leaves(version, &[key]);
-
-        // let existing_value = U256::from_big_endian(existing_leaf.leaf.value());
-        // if existing_value == value {
-        //     // we downgrade to read
-        //     // println!("Downgrading to read")
-        // } else {
-        // we write
         let mut tmp = [0u8; 32];
         value.to_big_endian(&mut tmp);
 
@@ -264,6 +282,7 @@ impl SnapshotExporter {
         let outfile = std::fs::OpenOptions::new()
             .write(true)
             .create(true)
+            .truncate(false)
             .open(path)?;
 
         serde_json::to_writer(outfile, &header)?;
@@ -304,6 +323,7 @@ impl SnapshotExporter {
         let mut outfile = std::fs::OpenOptions::new()
             .write(true)
             .create(true)
+            .truncate(false)
             .open(path)?;
 
         // Serialize chunk.
@@ -378,6 +398,7 @@ impl SnapshotExporter {
             let mut outfile = std::fs::OpenOptions::new()
                 .write(true)
                 .create(true)
+                .truncate(false)
                 .open(path)?;
 
             // Serialize chunk.

src/processor/snapshot/types.rs

@@ -1,5 +1,3 @@
-// FIXME:
-#![allow(dead_code)]
 use std::fmt;
 
 use chrono::{offset::Utc, DateTime};

src/processor/tree/tree_wrapper.rs

@@ -4,7 +4,9 @@ use blake2::{Blake2s256, Digest};
 use ethers::types::{Address, H256, U256};
 use eyre::Result;
 use state_reconstruct_fetcher::{
-    constants::storage::INITAL_STATE_PATH, database::InnerDB, types::CommitBlock,
+    constants::storage::INITAL_STATE_PATH,
+    database::InnerDB,
+    types::{v2::PackingType, CommitBlock},
 };
 use tokio::sync::Mutex;
 use zksync_merkle_tree::{Database, MerkleTree, RocksDBWrapper};
@@ -41,7 +43,7 @@ impl TreeWrapper {
             Vec::with_capacity(block.initial_storage_changes.len());
         for (key, value) in &block.initial_storage_changes {
             let key = U256::from_little_endian(key);
-            let value = H256::from(value);
+            let value = self.process_value(key, *value);
 
             key_value_pairs.push((key, value));
             self.snapshot
@@ -55,8 +57,13 @@ impl TreeWrapper {
         for (index, value) in &block.repeated_storage_changes {
             let index = *index;
             // Index is 1-based so we subtract 1.
-            let key = self.snapshot.lock().await.get_key(index - 1).unwrap();
-            let value = H256::from(value);
+            let key = self
+                .snapshot
+                .lock()
+                .await
+                .get_key(index - 1)
+                .expect("invalid key index");
+            let value = self.process_value(key, *value);
 
             key_value_pairs.push((key, value));
         }
@@ -75,6 +82,31 @@ impl TreeWrapper {
 
         root_hash
     }
+
+    fn process_value(&self, key: U256, value: PackingType) -> H256 {
+        let processed_value = match value {
+            PackingType::NoCompression(v) | PackingType::Transform(v) => v,
+            PackingType::Add(_) | PackingType::Sub(_) => {
+                let version = self.tree.latest_version().unwrap_or_default();
+                if let Ok(leaf) = self.tree.entries(version, &[key]) {
+                    let hash = leaf.last().unwrap().value_hash;
+                    let existing_value = U256::from(hash.to_fixed_bytes());
+                    // NOTE: We're explicitly allowing over-/underflow as per the spec.
+                    match value {
+                        PackingType::Add(v) => existing_value.overflowing_add(v).0,
+                        PackingType::Sub(v) => existing_value.overflowing_sub(v).0,
+                        _ => unreachable!(),
+                    }
+                } else {
+                    panic!("no key found for version")
+                }
+            }
+        };
+
+        let mut buffer = [0; 32];
+        processed_value.to_big_endian(&mut buffer);
+        H256::from(buffer)
+    }
 }
 
 /// Attempts to reconstruct the genesis state from a CSV file.
@@ -157,16 +189,6 @@ fn reconstruct_genesis_state<D: Database>(
     let mut key_value_pairs: Vec<(U256, H256)> = Vec::with_capacity(batched.len());
     for (address, key, value) in batched {
         let derived_key = derive_final_address_for_params(&address, &key);
-        // TODO: what to do here?
-        // let version = tree.latest_version().unwrap_or_default();
-        // let _leaf = tree.read_leaves(version, &[key]);
-
-        // let existing_value = U256::from_big_endian(existing_leaf.leaf.value());
-        // if existing_value == value {
-        //     // we downgrade to read
-        //     // println!("Downgrading to read")
-        // } else {
-        // we write
         let mut tmp = [0u8; 32];
         value.to_big_endian(&mut tmp);
 

state-reconstruct-fetcher/Cargo.toml

@@ -4,15 +4,19 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
+bincode = "1.3.3"
+blake2 = "0.10.6"
 ethers = "1.0.2"
 eyre = "0.6.8"
 indexmap = { version = "2.0.2", features = ["serde"] }
 rand = "0.8.5"
-rocksdb = "0.21"
 serde = { version = "1.0.189", features = ["derive"] }
 serde_json = { version = "1.0.107", features = ["std"] }
 serde_json_any_key = "2.0.0"
 thiserror = "1.0.50"
 tokio = { version = "1.33.0", features = ["signal"] }
 tokio-util = "0.7.10"
 tracing = "0.1.40"
+rocksdb = "0.21.0"
+hex = "0.4.3"
+chrono = "0.4.31"

state-reconstruct-fetcher/src/constants.rs

@@ -6,7 +6,7 @@ pub mod ethereum {
     pub const GENESIS_BLOCK: u64 = 16_627_460;
 
     /// Block number in Ethereum of the first Boojum-formatted block.
-    pub const BOOJUM_BLOCK: u64 = 18_711_784;
+    pub const BOOJUM_BLOCK: u64 = 18_715_403;
 
     /// zkSync smart contract address.
     pub const ZK_SYNC_ADDR: &str = "0x32400084C286CF3E17e7B677ea9583e60a000324";