From d89e3fa4ead0d18230fa8af378fa766c680cddda Mon Sep 17 00:00:00 2001 From: jackzhhuang Date: Thu, 9 Jan 2025 09:40:42 +0800 Subject: [PATCH 01/23] hint the root index whatever --- flexidag/src/blockdag.rs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/flexidag/src/blockdag.rs b/flexidag/src/blockdag.rs index 6ee2cf2404..9753c7de97 100644 --- a/flexidag/src/blockdag.rs +++ b/flexidag/src/blockdag.rs @@ -308,8 +308,20 @@ impl BlockDAG { } }; - if self.storage.reachability_store.read().get_reindex_root()? != header.pruning_point() - && header.pruning_point() != HashValue::zero() + if header.pruning_point() == HashValue::zero() { + info!( + "try to hint virtual selected parent, root index: {:?}", + self.storage.reachability_store.read().get_reindex_root() + ); + inquirer::hint_virtual_selected_parent( + self.storage.reachability_store.write().deref_mut(), + header.parent_hash(), + )?; + info!( + "after hint virtual selected parent, root index: {:?}", + self.storage.reachability_store.read().get_reindex_root() + ); + } else if self.storage.reachability_store.read().get_reindex_root()? != header.pruning_point() && self .storage .reachability_store From 64b76e3b2c8171e9a27b33254db3c21fed2c5e6d Mon Sep 17 00:00:00 2001 From: jackzhhuang Date: Thu, 9 Jan 2025 11:21:56 +0800 Subject: [PATCH 02/23] add config for lightweight config gap --- config/src/sync_config.rs | 20 +++++++++++++++++++ flexidag/src/blockdag.rs | 3 ++- flexidag/src/ghostdag/protocol.rs | 4 ++++ .../block_connector_service.rs | 19 +++++++++++++----- 4 files changed, 40 insertions(+), 6 deletions(-) diff --git a/config/src/sync_config.rs b/config/src/sync_config.rs index dc23345229..1a7e5c6642 100644 --- a/config/src/sync_config.rs +++ b/config/src/sync_config.rs @@ -28,6 +28,18 @@ pub struct SyncConfig { help = "max retry times once sync block failed, default 15." )] max_retry_times: Option, + + /// the maximum gap between the current head block's number and the peer's block's number + /// and if the block height broadcast by a peer node is greater than the height of the local head block by this maximum value, + /// a regular sync process will be initiated; + /// otherwise, a lightweight sync process will be triggered, strengthening the reference relationship between nodes. + #[serde(skip_serializing_if = "Option::is_none")] + #[clap( + name = "lightweight-sync-max-gap", + long, + help = "The height difference threshold for triggering a lightweight sync." + )] + lightweight_sync_max_gap: Option, } impl SyncConfig { @@ -38,6 +50,10 @@ impl SyncConfig { pub fn max_retry_times(&self) -> u64 { self.max_retry_times.unwrap_or(15) } + + pub fn lightweight_sync_max_gap(&self) -> Option { + self.lightweight_sync_max_gap + } } impl ConfigModule for SyncConfig { @@ -50,6 +66,10 @@ impl ConfigModule for SyncConfig { self.max_retry_times = opt.sync.max_retry_times; } + if opt.sync.lightweight_sync_max_gap.is_some() { + self.lightweight_sync_max_gap = opt.sync.lightweight_sync_max_gap; + } + Ok(()) } } diff --git a/flexidag/src/blockdag.rs b/flexidag/src/blockdag.rs index 9753c7de97..dc5c608161 100644 --- a/flexidag/src/blockdag.rs +++ b/flexidag/src/blockdag.rs @@ -321,7 +321,8 @@ impl BlockDAG { "after hint virtual selected parent, root index: {:?}", self.storage.reachability_store.read().get_reindex_root() ); - } else if self.storage.reachability_store.read().get_reindex_root()? != header.pruning_point() + } else if self.storage.reachability_store.read().get_reindex_root()? + != header.pruning_point() && self .storage .reachability_store diff --git a/flexidag/src/ghostdag/protocol.rs b/flexidag/src/ghostdag/protocol.rs index 435b09b421..72ad2006b2 100644 --- a/flexidag/src/ghostdag/protocol.rs +++ b/flexidag/src/ghostdag/protocol.rs @@ -485,6 +485,10 @@ impl< }); Ok(sorted_blocks) } + + pub fn k(&self) -> KType { + self.k + } } /// Chain block with attached ghostdag data diff --git a/sync/src/block_connector/block_connector_service.rs b/sync/src/block_connector/block_connector_service.rs index df3162e8ff..25a63615ca 100644 --- a/sync/src/block_connector/block_connector_service.rs +++ b/sync/src/block_connector/block_connector_service.rs @@ -16,7 +16,6 @@ use anyhow::{bail, format_err, Ok, Result}; use network_api::PeerProvider; use starcoin_chain::BlockChain; use starcoin_chain_api::{ChainReader, ConnectBlockError, WriteableChainService}; -use starcoin_config::genesis_config::G_BASE_MAX_UNCLES_PER_BLOCK; use starcoin_config::{NodeConfig, G_CRATE_VERSION}; use starcoin_consensus::Consensus; use starcoin_crypto::HashValue; @@ -137,10 +136,20 @@ where return false; } let gap = current_number.saturating_sub(block_header.number()); - if gap <= G_BASE_MAX_UNCLES_PER_BLOCK.saturating_mul(2) { - return true; - } - false + let k = self.chain_service.get_dag().ghost_dag_manager().k() as u64; + let config_gap = self + .config + .sync + .lightweight_sync_max_gap() + .map_or(k.saturating_mul(2), |max_gap| max_gap); + debug!( + "is-near-block: current_number: {:?}, block_number: {:?}, gap: {:?}, config_gap: {:?}", + current_number, + block_header.number(), + gap, + config_gap + ); + gap <= config_gap } } From 208a93cba9ea4d1dec48b5d3e4fbe2e393941b85 Mon Sep 17 00:00:00 2001 From: jackzhhuang Date: Thu, 9 Jan 2025 14:08:31 +0800 Subject: [PATCH 03/23] add limit on parents count --- config/src/miner_config.rs | 12 ++++++++++ flexidag/src/blockdag.rs | 23 ++++++++++++++++--- .../block_connector_service.rs | 1 + 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/config/src/miner_config.rs b/config/src/miner_config.rs index 582042259c..7eee643322 100644 --- a/config/src/miner_config.rs +++ b/config/src/miner_config.rs @@ -34,6 +34,10 @@ pub struct MinerConfig { #[serde(skip)] #[clap(skip)] base: Option>, + + #[serde(skip_serializing_if = "Option::is_none")] + #[clap(long = "maximum-parents_count")] + pub maximum_parents_count: Option, } impl MinerConfig { @@ -60,6 +64,10 @@ impl MinerConfig { enable_stderr: true, }) } + + pub fn maximum_parents_count(&self) -> u64 { + self.maximum_parents_count.unwrap_or_else(|| 16) + } } #[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)] @@ -103,6 +111,10 @@ impl ConfigModule for MinerConfig { self.block_gas_limit = opt.miner.block_gas_limit; } + if opt.miner.maximum_parents_count.is_some() { + self.maximum_parents_count = opt.miner.maximum_parents_count; + } + Ok(()) } } diff --git a/flexidag/src/blockdag.rs b/flexidag/src/blockdag.rs index dc5c608161..46d9c8223d 100644 --- a/flexidag/src/blockdag.rs +++ b/flexidag/src/blockdag.rs @@ -19,6 +19,7 @@ use crate::process_key_already_error; use crate::prune::pruning_point_manager::PruningPointManagerT; use crate::reachability::ReachabilityError; use anyhow::{bail, ensure, Ok}; +use itertools::Itertools; use parking_lot::Mutex; use rocksdb::WriteBatch; use starcoin_config::temp_dir; @@ -322,8 +323,7 @@ impl BlockDAG { self.storage.reachability_store.read().get_reindex_root() ); } else if self.storage.reachability_store.read().get_reindex_root()? - != header.pruning_point() - && self + != header.pruning_point() && self .storage .reachability_store .read() @@ -642,9 +642,24 @@ impl BlockDAG { previous_ghostdata: &GhostdagData, pruning_depth: u64, pruning_finality: u64, + max_parents_count: u64, ) -> anyhow::Result { info!("start to calculate the mergeset and tips, previous pruning point: {:?}, previous ghostdata: {:?} and its red block count: {:?}", previous_pruning_point, previous_ghostdata.to_compact(), previous_ghostdata.mergeset_reds.len()); - let dag_state = self.get_dag_state(previous_pruning_point)?; + let mut dag_state = self.get_dag_state(previous_pruning_point)?; + + // filter + if dag_state.tips.len() > max_parents_count as usize { + dag_state.tips = dag_state.tips.into_iter().sorted_by(|a, b| { + let a_blue_work = self.storage.ghost_dag_store.get_blue_work(*a).expect(&format!("the ghostdag data should be existed for {:?}", a)); + let b_blue_work = self.storage.ghost_dag_store.get_blue_work(*a).expect(&format!("the ghostdag data should be existed for {:?}", b)); + if a_blue_work == b_blue_work { + a.cmp(b) + } else { + b_blue_work.cmp(&a_blue_work) + } + }).take(max_parents_count as usize).collect(); + } + let next_ghostdata = self.ghostdata(&dag_state.tips)?; info!( "start to calculate the mergeset and tips for tips: {:?}, and last pruning point: {:?} and next ghostdata: {:?}, red block count: {:?}", @@ -657,10 +672,12 @@ impl BlockDAG { pruning_depth, pruning_finality, )?; + info!( "the next pruning point is: {:?}, and the previous pruning point is: {:?}", next_pruning_point, previous_pruning_point ); + if next_pruning_point == Hash::zero() || next_pruning_point == previous_pruning_point { anyhow::Ok(MineNewDagBlockInfo { tips: dag_state.tips, diff --git a/sync/src/block_connector/block_connector_service.rs b/sync/src/block_connector/block_connector_service.rs index 25a63615ca..e3e65151bc 100644 --- a/sync/src/block_connector/block_connector_service.rs +++ b/sync/src/block_connector/block_connector_service.rs @@ -453,6 +453,7 @@ where previous_ghostdata.as_ref(), pruning_depth, pruning_finality, + self.config.miner.maximum_parents_count(), )? } else { let genesis = ctx.get_shared::()?; From bcf59704794a2154dc4de6fb6ef0903a8ced82b4 Mon Sep 17 00:00:00 2001 From: jackzhhuang Date: Thu, 9 Jan 2025 14:35:29 +0800 Subject: [PATCH 04/23] add config for parents count --- chain/mock/src/mock_chain.rs | 2 ++ config/src/lib.rs | 2 +- config/src/miner_config.rs | 4 +++- flexidag/src/blockdag.rs | 46 +++++++++++++++++++++++++++--------- flexidag/tests/tests.rs | 3 +++ 5 files changed, 44 insertions(+), 13 deletions(-) diff --git a/chain/mock/src/mock_chain.rs b/chain/mock/src/mock_chain.rs index c11e8c9766..4032073f11 100644 --- a/chain/mock/src/mock_chain.rs +++ b/chain/mock/src/mock_chain.rs @@ -4,6 +4,7 @@ use anyhow::{format_err, Result}; use starcoin_account_api::AccountInfo; use starcoin_chain::{BlockChain, ChainReader, ChainWriter}; +use starcoin_config::miner_config::G_MAX_PARENTS_COUNT; use starcoin_config::ChainNetwork; use starcoin_consensus::Consensus; use starcoin_crypto::HashValue; @@ -266,6 +267,7 @@ impl MockChain { prevous_ghostdata.as_ref(), 4, 3, + G_MAX_PARENTS_COUNT, )?; debug!( diff --git a/config/src/lib.rs b/config/src/lib.rs index 491edce6b9..3bd5815186 100644 --- a/config/src/lib.rs +++ b/config/src/lib.rs @@ -33,7 +33,7 @@ pub mod genesis_config; mod helper; mod logger_config; mod metrics_config; -mod miner_config; +pub mod miner_config; mod network_config; mod rpc_config; mod storage_config; diff --git a/config/src/miner_config.rs b/config/src/miner_config.rs index 7eee643322..e69f54bdbb 100644 --- a/config/src/miner_config.rs +++ b/config/src/miner_config.rs @@ -7,6 +7,8 @@ use clap::Parser; use serde::{Deserialize, Serialize}; use std::sync::Arc; +pub static G_MAX_PARENTS_COUNT: u64 = 16; + #[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize, Parser)] #[serde(deny_unknown_fields)] pub struct MinerConfig { @@ -66,7 +68,7 @@ impl MinerConfig { } pub fn maximum_parents_count(&self) -> u64 { - self.maximum_parents_count.unwrap_or_else(|| 16) + self.maximum_parents_count.unwrap_or(G_MAX_PARENTS_COUNT) } } diff --git a/flexidag/src/blockdag.rs b/flexidag/src/blockdag.rs index 46d9c8223d..72233d31bc 100644 --- a/flexidag/src/blockdag.rs +++ b/flexidag/src/blockdag.rs @@ -323,7 +323,8 @@ impl BlockDAG { self.storage.reachability_store.read().get_reindex_root() ); } else if self.storage.reachability_store.read().get_reindex_root()? - != header.pruning_point() && self + != header.pruning_point() + && self .storage .reachability_store .read() @@ -647,17 +648,40 @@ impl BlockDAG { info!("start to calculate the mergeset and tips, previous pruning point: {:?}, previous ghostdata: {:?} and its red block count: {:?}", previous_pruning_point, previous_ghostdata.to_compact(), previous_ghostdata.mergeset_reds.len()); let mut dag_state = self.get_dag_state(previous_pruning_point)?; - // filter + // filter if dag_state.tips.len() > max_parents_count as usize { - dag_state.tips = dag_state.tips.into_iter().sorted_by(|a, b| { - let a_blue_work = self.storage.ghost_dag_store.get_blue_work(*a).expect(&format!("the ghostdag data should be existed for {:?}", a)); - let b_blue_work = self.storage.ghost_dag_store.get_blue_work(*a).expect(&format!("the ghostdag data should be existed for {:?}", b)); - if a_blue_work == b_blue_work { - a.cmp(b) - } else { - b_blue_work.cmp(&a_blue_work) - } - }).take(max_parents_count as usize).collect(); + dag_state.tips = dag_state + .tips + .into_iter() + .sorted_by(|a, b| { + let a_blue_work = self + .storage + .ghost_dag_store + .get_blue_work(*a) + .unwrap_or_else(|e| { + panic!( + "the ghostdag data should be existed for {:?}, e: {:?}", + a, e + ) + }); + let b_blue_work = self + .storage + .ghost_dag_store + .get_blue_work(*a) + .unwrap_or_else(|e| { + panic!( + "the ghostdag data should be existed for {:?}, e: {:?}", + b, e + ) + }); + if a_blue_work == b_blue_work { + a.cmp(b) + } else { + b_blue_work.cmp(&a_blue_work) + } + }) + .take(max_parents_count as usize) + .collect(); } let next_ghostdata = self.ghostdata(&dag_state.tips)?; diff --git a/flexidag/tests/tests.rs b/flexidag/tests/tests.rs index d9ddde5b34..d08dbcb63f 100644 --- a/flexidag/tests/tests.rs +++ b/flexidag/tests/tests.rs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 use anyhow::{bail, format_err, Ok, Result}; +use starcoin_config::miner_config::G_MAX_PARENTS_COUNT; use starcoin_crypto::HashValue as Hash; use starcoin_dag::{ blockdag::{BlockDAG, MineNewDagBlockInfo}, @@ -1028,6 +1029,7 @@ fn test_prune() -> anyhow::Result<()> { previous_ghostdata.as_ref(), pruning_depth, pruning_finality, + G_MAX_PARENTS_COUNT, )?; assert_eq!(pruning_point, block_main_2.id()); @@ -1056,6 +1058,7 @@ fn test_prune() -> anyhow::Result<()> { previous_ghostdata.as_ref(), pruning_depth, pruning_finality, + G_MAX_PARENTS_COUNT, )?; assert_eq!(pruning_point, block_main_2.id()); From 019d0a5b2614fd5b248675c0ab47e0a3c548c782 Mon Sep 17 00:00:00 2001 From: jackzhhuang Date: Fri, 10 Jan 2025 00:03:16 +0800 Subject: [PATCH 05/23] ignore hint failed --- flexidag/src/blockdag.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flexidag/src/blockdag.rs b/flexidag/src/blockdag.rs index 72233d31bc..624a0ca381 100644 --- a/flexidag/src/blockdag.rs +++ b/flexidag/src/blockdag.rs @@ -314,10 +314,10 @@ impl BlockDAG { "try to hint virtual selected parent, root index: {:?}", self.storage.reachability_store.read().get_reindex_root() ); - inquirer::hint_virtual_selected_parent( + let _ = inquirer::hint_virtual_selected_parent( self.storage.reachability_store.write().deref_mut(), header.parent_hash(), - )?; + ); info!( "after hint virtual selected parent, root index: {:?}", self.storage.reachability_store.read().get_reindex_root() From 4351989679ae17fab4fe82179674ba953d0d5a33 Mon Sep 17 00:00:00 2001 From: jackzhhuang Date: Mon, 13 Jan 2025 00:49:53 +0800 Subject: [PATCH 06/23] use false as set_sync's argument --- flexidag/src/consensusdb/db.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexidag/src/consensusdb/db.rs b/flexidag/src/consensusdb/db.rs index 72632d11db..96c846c534 100644 --- a/flexidag/src/consensusdb/db.rs +++ b/flexidag/src/consensusdb/db.rs @@ -104,7 +104,7 @@ impl FlexiDagStorage { } pub fn write_batch(&self, batch: WriteBatch) -> Result<(), StoreError> { - self.db.raw_write_batch_sync(batch).map_err(|e| { + self.db.raw_write_batch(batch).map_err(|e| { StoreError::DBIoError(format!( "failed to write in batch for dag data, error: {:?}", e.to_string() From 0d3aa2433122aa07e5c6a953360d86ae04e411c2 Mon Sep 17 00:00:00 2001 From: jackzhhuang Date: Mon, 13 Jan 2025 09:33:02 +0800 Subject: [PATCH 07/23] add some info for debug --- flexidag/src/blockdag.rs | 36 ++++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/flexidag/src/blockdag.rs b/flexidag/src/blockdag.rs index 624a0ca381..0e150666aa 100644 --- a/flexidag/src/blockdag.rs +++ b/flexidag/src/blockdag.rs @@ -103,8 +103,14 @@ impl BlockDAG { } pub fn has_block_connected(&self, block_header: &BlockHeader) -> anyhow::Result { - let _ghostdata = match self.storage.ghost_dag_store.get_data(block_header.id()) { - std::result::Result::Ok(data) => data, + match self.storage.ghost_dag_store.has(block_header.id()) { + std::result::Result::Ok(true) => (), + std::result::Result::Ok(false) =>{ + warn!( + "failed to get ghostdata by hash, the block should be re-executed", + ); + return anyhow::Result::Ok(false); + } Err(e) => { warn!( "failed to get ghostdata by hash: {:?}, the block should be re-executed", @@ -114,8 +120,14 @@ impl BlockDAG { } }; - let _dag_header = match self.storage.header_store.get_header(block_header.id()) { - std::result::Result::Ok(header) => header, + match self.storage.header_store.has(block_header.id()) { + std::result::Result::Ok(true) => (), + std::result::Result::Ok(false) => { + warn!( + "failed to get header by hash, the block should be re-executed", + ); + return anyhow::Result::Ok(false) + } Err(e) => { warn!( "failed to get header by hash: {:?}, the block should be re-executed", @@ -359,6 +371,8 @@ impl BlockDAG { self.storage.reachability_store.upgradable_read(), ); + info!("start to commit via batch1, header id: {:?}", header.id()); + // Store ghostdata process_key_already_error(self.storage.ghost_dag_store.insert_batch( &mut batch, @@ -367,6 +381,8 @@ impl BlockDAG { )) .expect("failed to ghostdata in batch"); + info!("start to commit via batch2, header id: {:?}", header.id()); + // Update reachability store debug!( "start to update reachability data for block: {:?}, number: {:?}", @@ -380,6 +396,8 @@ impl BlockDAG { .collect::>() .into_iter(); + info!("start to commit via batch3, header id: {:?}, count of mergeset: {:?}, ", header.id(), merge_set.len()); + match inquirer::add_block( &mut stage, header.id(), @@ -401,6 +419,8 @@ impl BlockDAG { }, } + info!("start to commit via batch4, header id: {:?}", header.id()); + process_key_already_error(self.storage.relations_store.write().insert_batch( &mut batch, header.id(), @@ -408,6 +428,8 @@ impl BlockDAG { )) .expect("failed to insert relations in batch"); + info!("start to commit via batch5, header id: {:?}", header.id()); + // Store header store process_key_already_error(self.storage.header_store.insert( header.id(), @@ -416,6 +438,8 @@ impl BlockDAG { )) .expect("failed to insert header in batch"); + info!("start to commit via batch6, header id: {:?}", header.id()); + // the read lock will be updated to the write lock // and then write the batch // and then release the lock @@ -423,11 +447,15 @@ impl BlockDAG { .commit(&mut batch) .expect("failed to write the stage reachability in batch"); + info!("start to commit via batch7, header id: {:?}", header.id()); + // write the data just one time self.storage .write_batch(batch) .expect("failed to write dag data in batch"); + info!("start to commit via batch8, header id: {:?}", header.id()); + drop(lock_guard); info!("finish writing the batch, head id: {:?}", header.id()); From 07f6f1bb218fa3e68e45a3e8306e2f5876041ee2 Mon Sep 17 00:00:00 2001 From: jackzhhuang Date: Mon, 13 Jan 2025 12:13:09 +0800 Subject: [PATCH 08/23] hint whatever --- flexidag/src/blockdag.rs | 61 ++++++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/flexidag/src/blockdag.rs b/flexidag/src/blockdag.rs index 0e150666aa..f5b22f10e5 100644 --- a/flexidag/src/blockdag.rs +++ b/flexidag/src/blockdag.rs @@ -105,10 +105,8 @@ impl BlockDAG { pub fn has_block_connected(&self, block_header: &BlockHeader) -> anyhow::Result { match self.storage.ghost_dag_store.has(block_header.id()) { std::result::Result::Ok(true) => (), - std::result::Result::Ok(false) =>{ - warn!( - "failed to get ghostdata by hash, the block should be re-executed", - ); + std::result::Result::Ok(false) => { + warn!("failed to get ghostdata by hash, the block should be re-executed",); return anyhow::Result::Ok(false); } Err(e) => { @@ -123,10 +121,8 @@ impl BlockDAG { match self.storage.header_store.has(block_header.id()) { std::result::Result::Ok(true) => (), std::result::Result::Ok(false) => { - warn!( - "failed to get header by hash, the block should be re-executed", - ); - return anyhow::Result::Ok(false) + warn!("failed to get header by hash, the block should be re-executed",); + return anyhow::Result::Ok(false); } Err(e) => { warn!( @@ -396,7 +392,11 @@ impl BlockDAG { .collect::>() .into_iter(); - info!("start to commit via batch3, header id: {:?}, count of mergeset: {:?}, ", header.id(), merge_set.len()); + info!( + "start to commit via batch3, header id: {:?}, count of mergeset: {:?}, ", + header.id(), + merge_set.len() + ); match inquirer::add_block( &mut stage, @@ -489,27 +489,28 @@ impl BlockDAG { Some(ghostdata) => ghostdata, }; - if self.storage.reachability_store.read().get_reindex_root()? != header.pruning_point() - && header.pruning_point() != HashValue::zero() - && self - .storage - .reachability_store - .read() - .has(header.pruning_point())? - { - info!( - "try to hint virtual selected parent, root index: {:?}", - self.storage.reachability_store.read().get_reindex_root() - ); - inquirer::hint_virtual_selected_parent( - self.storage.reachability_store.write().deref_mut(), - header.pruning_point(), - )?; - info!( - "after hint virtual selected parent, root index: {:?}", - self.storage.reachability_store.read().get_reindex_root() - ); - } + // if self.storage.reachability_store.read().get_reindex_root()? != header.pruning_point() + // && header.pruning_point() != HashValue::zero() + // && self + // .storage + // .reachability_store + // .read() + // .has(header.pruning_point())? + // { + info!( + "try to hint virtual selected parent, root index: {:?}", + self.storage.reachability_store.read().get_reindex_root() + ); + let hint_result = inquirer::hint_virtual_selected_parent( + self.storage.reachability_store.write().deref_mut(), + header.parent_hash(), + ); + info!( + "after hint virtual selected parent, root index: {:?}, hint result: {:?}", + self.storage.reachability_store.read().get_reindex_root(), + hint_result + ); + // } info!("start to commit via batch, header id: {:?}", header.id()); From 88bd7c96658c75ae6722481a64bc6648f971d0d6 Mon Sep 17 00:00:00 2001 From: jackzhhuang Date: Mon, 13 Jan 2025 12:38:32 +0800 Subject: [PATCH 09/23] hint whatever --- flexidag/src/blockdag.rs | 96 ++++++++++++++++++++-------------------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/flexidag/src/blockdag.rs b/flexidag/src/blockdag.rs index f5b22f10e5..4871a47319 100644 --- a/flexidag/src/blockdag.rs +++ b/flexidag/src/blockdag.rs @@ -105,8 +105,10 @@ impl BlockDAG { pub fn has_block_connected(&self, block_header: &BlockHeader) -> anyhow::Result { match self.storage.ghost_dag_store.has(block_header.id()) { std::result::Result::Ok(true) => (), - std::result::Result::Ok(false) => { - warn!("failed to get ghostdata by hash, the block should be re-executed",); + std::result::Result::Ok(false) =>{ + warn!( + "failed to get ghostdata by hash, the block should be re-executed", + ); return anyhow::Result::Ok(false); } Err(e) => { @@ -121,8 +123,10 @@ impl BlockDAG { match self.storage.header_store.has(block_header.id()) { std::result::Result::Ok(true) => (), std::result::Result::Ok(false) => { - warn!("failed to get header by hash, the block should be re-executed",); - return anyhow::Result::Ok(false); + warn!( + "failed to get header by hash, the block should be re-executed", + ); + return anyhow::Result::Ok(false) } Err(e) => { warn!( @@ -317,40 +321,41 @@ impl BlockDAG { } }; - if header.pruning_point() == HashValue::zero() { - info!( - "try to hint virtual selected parent, root index: {:?}", - self.storage.reachability_store.read().get_reindex_root() - ); - let _ = inquirer::hint_virtual_selected_parent( - self.storage.reachability_store.write().deref_mut(), - header.parent_hash(), - ); - info!( - "after hint virtual selected parent, root index: {:?}", - self.storage.reachability_store.read().get_reindex_root() - ); - } else if self.storage.reachability_store.read().get_reindex_root()? - != header.pruning_point() - && self - .storage - .reachability_store - .read() - .has(header.pruning_point())? - { + // if header.pruning_point() == HashValue::zero() { + // info!( + // "try to hint virtual selected parent, root index: {:?}", + // self.storage.reachability_store.read().get_reindex_root() + // ); + // let _ = inquirer::hint_virtual_selected_parent( + // self.storage.reachability_store.write().deref_mut(), + // header.parent_hash(), + // ); + // info!( + // "after hint virtual selected parent, root index: {:?}", + // self.storage.reachability_store.read().get_reindex_root() + // ); + // } else if self.storage.reachability_store.read().get_reindex_root()? + // != header.pruning_point() + // && self + // .storage + // .reachability_store + // .read() + // .has(header.pruning_point())? + // { info!( "try to hint virtual selected parent, root index: {:?}", self.storage.reachability_store.read().get_reindex_root() ); - inquirer::hint_virtual_selected_parent( + let hint_result = inquirer::hint_virtual_selected_parent( self.storage.reachability_store.write().deref_mut(), header.pruning_point(), - )?; + ); info!( - "after hint virtual selected parent, root index: {:?}", - self.storage.reachability_store.read().get_reindex_root() + "after hint virtual selected parent, root index: {:?}, hint result: {:?}", + self.storage.reachability_store.read().get_reindex_root(), + hint_result ); - } + // } // Create a DB batch writer let mut batch = WriteBatch::default(); @@ -392,11 +397,7 @@ impl BlockDAG { .collect::>() .into_iter(); - info!( - "start to commit via batch3, header id: {:?}, count of mergeset: {:?}, ", - header.id(), - merge_set.len() - ); + info!("start to commit via batch3, header id: {:?}, count of mergeset: {:?}, ", header.id(), merge_set.len()); match inquirer::add_block( &mut stage, @@ -497,19 +498,18 @@ impl BlockDAG { // .read() // .has(header.pruning_point())? // { - info!( - "try to hint virtual selected parent, root index: {:?}", - self.storage.reachability_store.read().get_reindex_root() - ); - let hint_result = inquirer::hint_virtual_selected_parent( - self.storage.reachability_store.write().deref_mut(), - header.parent_hash(), - ); - info!( - "after hint virtual selected parent, root index: {:?}, hint result: {:?}", - self.storage.reachability_store.read().get_reindex_root(), - hint_result - ); + info!( + "try to hint virtual selected parent, root index: {:?}", + self.storage.reachability_store.read().get_reindex_root() + ); + let hint_result = inquirer::hint_virtual_selected_parent( + self.storage.reachability_store.write().deref_mut(), + header.parent_hash(), + ); + info!( + "after hint virtual selected parent, root index: {:?}, hint result: {:?}", + self.storage.reachability_store.read().get_reindex_root(), hint_result + ); // } info!("start to commit via batch, header id: {:?}", header.id()); From 82129683066d86853072d5cacabe1733cc58fdf4 Mon Sep 17 00:00:00 2001 From: jackzhhuang Date: Mon, 13 Jan 2025 16:46:52 +0800 Subject: [PATCH 10/23] check the historical block before verifying --- chain/src/chain.rs | 15 ++++++- flexidag/src/blockdag.rs | 97 +++++++++++++++++++++++++--------------- 2 files changed, 74 insertions(+), 38 deletions(-) diff --git a/chain/src/chain.rs b/chain/src/chain.rs index 4b72772df0..acddd4de70 100644 --- a/chain/src/chain.rs +++ b/chain/src/chain.rs @@ -1381,7 +1381,20 @@ impl ChainReader for BlockChain { uncles: &[BlockHeader], header: &BlockHeader, ) -> Result { - Ok(self.dag().verify_and_ghostdata(uncles, header)?) + let latest_pruning_point = { + match self.storage.get_startup_info().unwrap_or(None) { + Some(startup_info) => self + .storage + .get_block_header_by_hash(startup_info.main) + .unwrap_or(None) + .map(|head_block| head_block.pruning_point()), + None => None, + } + }; + + Ok(self + .dag() + .verify_and_ghostdata(uncles, header, latest_pruning_point)?) } fn is_dag_ancestor_of(&self, ancestor: HashValue, descendant: HashValue) -> Result { diff --git a/flexidag/src/blockdag.rs b/flexidag/src/blockdag.rs index 4871a47319..bdee720341 100644 --- a/flexidag/src/blockdag.rs +++ b/flexidag/src/blockdag.rs @@ -105,10 +105,8 @@ impl BlockDAG { pub fn has_block_connected(&self, block_header: &BlockHeader) -> anyhow::Result { match self.storage.ghost_dag_store.has(block_header.id()) { std::result::Result::Ok(true) => (), - std::result::Result::Ok(false) =>{ - warn!( - "failed to get ghostdata by hash, the block should be re-executed", - ); + std::result::Result::Ok(false) => { + warn!("failed to get ghostdata by hash, the block should be re-executed",); return anyhow::Result::Ok(false); } Err(e) => { @@ -123,10 +121,8 @@ impl BlockDAG { match self.storage.header_store.has(block_header.id()) { std::result::Result::Ok(true) => (), std::result::Result::Ok(false) => { - warn!( - "failed to get header by hash, the block should be re-executed", - ); - return anyhow::Result::Ok(false) + warn!("failed to get header by hash, the block should be re-executed",); + return anyhow::Result::Ok(false); } Err(e) => { warn!( @@ -342,19 +338,19 @@ impl BlockDAG { // .read() // .has(header.pruning_point())? // { - info!( - "try to hint virtual selected parent, root index: {:?}", - self.storage.reachability_store.read().get_reindex_root() - ); - let hint_result = inquirer::hint_virtual_selected_parent( - self.storage.reachability_store.write().deref_mut(), - header.pruning_point(), - ); - info!( - "after hint virtual selected parent, root index: {:?}, hint result: {:?}", - self.storage.reachability_store.read().get_reindex_root(), - hint_result - ); + info!( + "try to hint virtual selected parent, root index: {:?}", + self.storage.reachability_store.read().get_reindex_root() + ); + let hint_result = inquirer::hint_virtual_selected_parent( + self.storage.reachability_store.write().deref_mut(), + header.pruning_point(), + ); + info!( + "after hint virtual selected parent, root index: {:?}, hint result: {:?}", + self.storage.reachability_store.read().get_reindex_root(), + hint_result + ); // } // Create a DB batch writer @@ -397,7 +393,11 @@ impl BlockDAG { .collect::>() .into_iter(); - info!("start to commit via batch3, header id: {:?}, count of mergeset: {:?}, ", header.id(), merge_set.len()); + info!( + "start to commit via batch3, header id: {:?}, count of mergeset: {:?}, ", + header.id(), + merge_set.len() + ); match inquirer::add_block( &mut stage, @@ -498,18 +498,19 @@ impl BlockDAG { // .read() // .has(header.pruning_point())? // { - info!( - "try to hint virtual selected parent, root index: {:?}", - self.storage.reachability_store.read().get_reindex_root() - ); - let hint_result = inquirer::hint_virtual_selected_parent( - self.storage.reachability_store.write().deref_mut(), - header.parent_hash(), - ); - info!( - "after hint virtual selected parent, root index: {:?}, hint result: {:?}", - self.storage.reachability_store.read().get_reindex_root(), hint_result - ); + info!( + "try to hint virtual selected parent, root index: {:?}", + self.storage.reachability_store.read().get_reindex_root() + ); + let hint_result = inquirer::hint_virtual_selected_parent( + self.storage.reachability_store.write().deref_mut(), + header.parent_hash(), + ); + info!( + "after hint virtual selected parent, root index: {:?}, hint result: {:?}", + self.storage.reachability_store.read().get_reindex_root(), + hint_result + ); // } info!("start to commit via batch, header id: {:?}", header.id()); @@ -794,18 +795,40 @@ impl BlockDAG { self.pruning_point_manager().reachability_service() } + fn check_historical_block( + &self, + header: &BlockHeader, + latest_pruning_point: Option, + ) -> Result { + if let Some(pruning_point) = latest_pruning_point { + if pruning_point == HashValue::zero() { + Ok(true) + } else if header.pruning_point() == pruning_point { + Ok(false) + } else if self.check_ancestor_of(header.pruning_point(), pruning_point)? { + Ok(true) + } else { + Ok(false) + } + } else { + Ok(true) + } + } + pub fn verify_and_ghostdata( &self, blue_blocks: &[BlockHeader], header: &BlockHeader, + latest_pruning_point: Option, ) -> Result { - if header.pruning_point() != HashValue::zero() { - self.ghost_dag_manager().ghostdag(&header.parents()) - } else { + if self.check_historical_block(header, latest_pruning_point)? { self.ghost_dag_manager() .verify_and_ghostdata(blue_blocks, header) + } else { + self.ghost_dag_manager().ghostdag(&header.parents()) } } + pub fn check_upgrade(&self, main: &BlockHeader, genesis_id: HashValue) -> anyhow::Result<()> { // set the state with key 0 if main.version() == 0 || main.version() == 1 { From b8935ed4796ea885573c51bb2c339be1b31ad0d9 Mon Sep 17 00:00:00 2001 From: jackzhhuang Date: Mon, 13 Jan 2025 17:09:52 +0800 Subject: [PATCH 11/23] recover hint logic --- flexidag/src/blockdag.rs | 121 +++++++++++++++++++++------------------ 1 file changed, 64 insertions(+), 57 deletions(-) diff --git a/flexidag/src/blockdag.rs b/flexidag/src/blockdag.rs index bdee720341..3eab8c2037 100644 --- a/flexidag/src/blockdag.rs +++ b/flexidag/src/blockdag.rs @@ -317,41 +317,41 @@ impl BlockDAG { } }; - // if header.pruning_point() == HashValue::zero() { - // info!( - // "try to hint virtual selected parent, root index: {:?}", - // self.storage.reachability_store.read().get_reindex_root() - // ); - // let _ = inquirer::hint_virtual_selected_parent( - // self.storage.reachability_store.write().deref_mut(), - // header.parent_hash(), - // ); - // info!( - // "after hint virtual selected parent, root index: {:?}", - // self.storage.reachability_store.read().get_reindex_root() - // ); - // } else if self.storage.reachability_store.read().get_reindex_root()? - // != header.pruning_point() - // && self - // .storage - // .reachability_store - // .read() - // .has(header.pruning_point())? - // { - info!( - "try to hint virtual selected parent, root index: {:?}", - self.storage.reachability_store.read().get_reindex_root() - ); - let hint_result = inquirer::hint_virtual_selected_parent( - self.storage.reachability_store.write().deref_mut(), - header.pruning_point(), - ); - info!( - "after hint virtual selected parent, root index: {:?}, hint result: {:?}", - self.storage.reachability_store.read().get_reindex_root(), - hint_result - ); - // } + if header.pruning_point() == HashValue::zero() { + info!( + "try to hint virtual selected parent, root index: {:?}", + self.storage.reachability_store.read().get_reindex_root() + ); + let _ = inquirer::hint_virtual_selected_parent( + self.storage.reachability_store.write().deref_mut(), + header.parent_hash(), + ); + info!( + "after hint virtual selected parent, root index: {:?}", + self.storage.reachability_store.read().get_reindex_root() + ); + } else if self.storage.reachability_store.read().get_reindex_root()? + != header.pruning_point() + && self + .storage + .reachability_store + .read() + .has(header.pruning_point())? + { + info!( + "try to hint virtual selected parent, root index: {:?}", + self.storage.reachability_store.read().get_reindex_root() + ); + let hint_result = inquirer::hint_virtual_selected_parent( + self.storage.reachability_store.write().deref_mut(), + header.pruning_point(), + ); + info!( + "after hint virtual selected parent, root index: {:?}, hint result: {:?}", + self.storage.reachability_store.read().get_reindex_root(), + hint_result + ); + } // Create a DB batch writer let mut batch = WriteBatch::default(); @@ -490,28 +490,28 @@ impl BlockDAG { Some(ghostdata) => ghostdata, }; - // if self.storage.reachability_store.read().get_reindex_root()? != header.pruning_point() - // && header.pruning_point() != HashValue::zero() - // && self - // .storage - // .reachability_store - // .read() - // .has(header.pruning_point())? - // { - info!( - "try to hint virtual selected parent, root index: {:?}", - self.storage.reachability_store.read().get_reindex_root() - ); - let hint_result = inquirer::hint_virtual_selected_parent( - self.storage.reachability_store.write().deref_mut(), - header.parent_hash(), - ); - info!( - "after hint virtual selected parent, root index: {:?}, hint result: {:?}", - self.storage.reachability_store.read().get_reindex_root(), - hint_result - ); - // } + if self.storage.reachability_store.read().get_reindex_root()? != header.pruning_point() + && header.pruning_point() != HashValue::zero() + && self + .storage + .reachability_store + .read() + .has(header.pruning_point())? + { + info!( + "try to hint virtual selected parent, root index: {:?}", + self.storage.reachability_store.read().get_reindex_root() + ); + let hint_result = inquirer::hint_virtual_selected_parent( + self.storage.reachability_store.write().deref_mut(), + header.parent_hash(), + ); + info!( + "after hint virtual selected parent, root index: {:?}, hint result: {:?}", + self.storage.reachability_store.read().get_reindex_root(), + hint_result + ); + } info!("start to commit via batch, header id: {:?}", header.id()); @@ -802,12 +802,19 @@ impl BlockDAG { ) -> Result { if let Some(pruning_point) = latest_pruning_point { if pruning_point == HashValue::zero() { + info!("pruning point is zero"); Ok(true) } else if header.pruning_point() == pruning_point { + info!( + "pruning point is the same as the latest pruning point, pruning point: {:?}", + pruning_point + ); Ok(false) } else if self.check_ancestor_of(header.pruning_point(), pruning_point)? { + info!("pruning point is the ancestor of the latest pruning point, pruning point: {:?}, latest pruning point: {:?}", header.pruning_point(), pruning_point); Ok(true) } else { + info!("pruning point is not the ancestor of the latest pruning point, pruning point: {:?}, latest pruning point: {:?}", header.pruning_point(), pruning_point); Ok(false) } } else { From c7a7c223ed581db94b6693de0022e624e43c5555 Mon Sep 17 00:00:00 2001 From: jackzhhuang Date: Mon, 13 Jan 2025 17:22:30 +0800 Subject: [PATCH 12/23] remove some info --- flexidag/src/blockdag.rs | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/flexidag/src/blockdag.rs b/flexidag/src/blockdag.rs index 3eab8c2037..bd475990f2 100644 --- a/flexidag/src/blockdag.rs +++ b/flexidag/src/blockdag.rs @@ -368,8 +368,6 @@ impl BlockDAG { self.storage.reachability_store.upgradable_read(), ); - info!("start to commit via batch1, header id: {:?}", header.id()); - // Store ghostdata process_key_already_error(self.storage.ghost_dag_store.insert_batch( &mut batch, @@ -378,8 +376,6 @@ impl BlockDAG { )) .expect("failed to ghostdata in batch"); - info!("start to commit via batch2, header id: {:?}", header.id()); - // Update reachability store debug!( "start to update reachability data for block: {:?}, number: {:?}", @@ -393,12 +389,6 @@ impl BlockDAG { .collect::>() .into_iter(); - info!( - "start to commit via batch3, header id: {:?}, count of mergeset: {:?}, ", - header.id(), - merge_set.len() - ); - match inquirer::add_block( &mut stage, header.id(), @@ -420,8 +410,6 @@ impl BlockDAG { }, } - info!("start to commit via batch4, header id: {:?}", header.id()); - process_key_already_error(self.storage.relations_store.write().insert_batch( &mut batch, header.id(), @@ -429,8 +417,6 @@ impl BlockDAG { )) .expect("failed to insert relations in batch"); - info!("start to commit via batch5, header id: {:?}", header.id()); - // Store header store process_key_already_error(self.storage.header_store.insert( header.id(), @@ -439,8 +425,6 @@ impl BlockDAG { )) .expect("failed to insert header in batch"); - info!("start to commit via batch6, header id: {:?}", header.id()); - // the read lock will be updated to the write lock // and then write the batch // and then release the lock @@ -448,15 +432,11 @@ impl BlockDAG { .commit(&mut batch) .expect("failed to write the stage reachability in batch"); - info!("start to commit via batch7, header id: {:?}", header.id()); - // write the data just one time self.storage .write_batch(batch) .expect("failed to write dag data in batch"); - info!("start to commit via batch8, header id: {:?}", header.id()); - drop(lock_guard); info!("finish writing the batch, head id: {:?}", header.id()); From e4a7ad339a148f60f86929591a6f59c9dd2ce072 Mon Sep 17 00:00:00 2001 From: jackzhhuang Date: Thu, 16 Jan 2025 10:01:16 +0800 Subject: [PATCH 13/23] clean the info --- flexidag/src/blockdag.rs | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/flexidag/src/blockdag.rs b/flexidag/src/blockdag.rs index bd475990f2..fccf01e971 100644 --- a/flexidag/src/blockdag.rs +++ b/flexidag/src/blockdag.rs @@ -655,7 +655,6 @@ impl BlockDAG { pruning_finality: u64, max_parents_count: u64, ) -> anyhow::Result { - info!("start to calculate the mergeset and tips, previous pruning point: {:?}, previous ghostdata: {:?} and its red block count: {:?}", previous_pruning_point, previous_ghostdata.to_compact(), previous_ghostdata.mergeset_reds.len()); let mut dag_state = self.get_dag_state(previous_pruning_point)?; // filter @@ -695,10 +694,7 @@ impl BlockDAG { } let next_ghostdata = self.ghostdata(&dag_state.tips)?; - info!( - "start to calculate the mergeset and tips for tips: {:?}, and last pruning point: {:?} and next ghostdata: {:?}, red block count: {:?}", - dag_state.tips, previous_pruning_point, next_ghostdata.to_compact(), next_ghostdata.mergeset_reds.len() - ); + let next_pruning_point = self.pruning_point_manager().next_pruning_point( previous_pruning_point, previous_ghostdata, @@ -707,12 +703,11 @@ impl BlockDAG { pruning_finality, )?; - info!( - "the next pruning point is: {:?}, and the previous pruning point is: {:?}", - next_pruning_point, previous_pruning_point - ); - if next_pruning_point == Hash::zero() || next_pruning_point == previous_pruning_point { + info!( + "tips: {:?}, the next pruning point is: {:?}, the current ghostdata's selected parent: {:?}, blue blocks are: {:?} and its red blocks are: {:?}", + dag_state.tips, next_pruning_point, next_ghostdata.selected_parent, next_ghostdata.mergeset_blues, next_ghostdata.mergeset_reds.len(), + ); anyhow::Ok(MineNewDagBlockInfo { tips: dag_state.tips, blue_blocks: (*next_ghostdata.mergeset_blues).clone(), @@ -724,15 +719,13 @@ impl BlockDAG { previous_pruning_point, next_pruning_point, )?; - let mergeset_blues = (*self + let pruned_ghostdata = self .ghost_dag_manager() - .ghostdag(&pruned_tips)? - .mergeset_blues) - .clone(); + .ghostdag(&pruned_tips)?; + let mergeset_blues = pruned_ghostdata.mergeset_blues.as_ref().clone(); info!( - "previous tips are: {:?}, the pruned tips are: {:?}, the mergeset blues are: {:?}, the next pruning point is: {:?}", - dag_state.tips, - pruned_tips, mergeset_blues, next_pruning_point + "the pruning was triggered, previous tips: {:?}, the current tips: {:?}, the next pruning point is: {:?}, the current ghostdata's selected parent: {:?}, blue blocks are: {:?} and its red blocks are: {:?}", + pruned_tips, dag_state.tips, next_pruning_point, pruned_ghostdata.selected_parent, pruned_ghostdata.mergeset_blues, pruned_ghostdata.mergeset_reds.len(), ); anyhow::Ok(MineNewDagBlockInfo { tips: pruned_tips, From db63669f7a56083a55406c7a5309f2a896232eeb Mon Sep 17 00:00:00 2001 From: jackzhhuang Date: Thu, 16 Jan 2025 10:13:00 +0800 Subject: [PATCH 14/23] fix fmt --- flexidag/src/blockdag.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/flexidag/src/blockdag.rs b/flexidag/src/blockdag.rs index fccf01e971..09ad68138c 100644 --- a/flexidag/src/blockdag.rs +++ b/flexidag/src/blockdag.rs @@ -706,7 +706,7 @@ impl BlockDAG { if next_pruning_point == Hash::zero() || next_pruning_point == previous_pruning_point { info!( "tips: {:?}, the next pruning point is: {:?}, the current ghostdata's selected parent: {:?}, blue blocks are: {:?} and its red blocks are: {:?}", - dag_state.tips, next_pruning_point, next_ghostdata.selected_parent, next_ghostdata.mergeset_blues, next_ghostdata.mergeset_reds.len(), + dag_state.tips, next_pruning_point, next_ghostdata.selected_parent, next_ghostdata.mergeset_blues, next_ghostdata.mergeset_reds.len(), ); anyhow::Ok(MineNewDagBlockInfo { tips: dag_state.tips, @@ -719,13 +719,11 @@ impl BlockDAG { previous_pruning_point, next_pruning_point, )?; - let pruned_ghostdata = self - .ghost_dag_manager() - .ghostdag(&pruned_tips)?; + let pruned_ghostdata = self.ghost_dag_manager().ghostdag(&pruned_tips)?; let mergeset_blues = pruned_ghostdata.mergeset_blues.as_ref().clone(); info!( "the pruning was triggered, previous tips: {:?}, the current tips: {:?}, the next pruning point is: {:?}, the current ghostdata's selected parent: {:?}, blue blocks are: {:?} and its red blocks are: {:?}", - pruned_tips, dag_state.tips, next_pruning_point, pruned_ghostdata.selected_parent, pruned_ghostdata.mergeset_blues, pruned_ghostdata.mergeset_reds.len(), + pruned_tips, dag_state.tips, next_pruning_point, pruned_ghostdata.selected_parent, pruned_ghostdata.mergeset_blues, pruned_ghostdata.mergeset_reds.len(), ); anyhow::Ok(MineNewDagBlockInfo { tips: pruned_tips, From 9ac900664d9cacad7554016063ec493bf211e8ec Mon Sep 17 00:00:00 2001 From: jackzhhuang Date: Thu, 16 Jan 2025 10:15:58 +0800 Subject: [PATCH 15/23] fx sort tip logic and typo --- config/src/miner_config.rs | 2 +- flexidag/src/blockdag.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/src/miner_config.rs b/config/src/miner_config.rs index e69f54bdbb..1411905452 100644 --- a/config/src/miner_config.rs +++ b/config/src/miner_config.rs @@ -38,7 +38,7 @@ pub struct MinerConfig { base: Option>, #[serde(skip_serializing_if = "Option::is_none")] - #[clap(long = "maximum-parents_count")] + #[clap(long = "maximum-parents-count")] pub maximum_parents_count: Option, } diff --git a/flexidag/src/blockdag.rs b/flexidag/src/blockdag.rs index 09ad68138c..885bc4dc08 100644 --- a/flexidag/src/blockdag.rs +++ b/flexidag/src/blockdag.rs @@ -676,7 +676,7 @@ impl BlockDAG { let b_blue_work = self .storage .ghost_dag_store - .get_blue_work(*a) + .get_blue_work(*b) .unwrap_or_else(|e| { panic!( "the ghostdag data should be existed for {:?}, e: {:?}", From d2eaca8031ad6660132c65380d74c9222f95ea6a Mon Sep 17 00:00:00 2001 From: jackzhhuang Date: Thu, 16 Jan 2025 13:33:18 +0800 Subject: [PATCH 16/23] 1, when chekcing whether the block is historical, add logic for the main net 2, add checking for the first pruning point --- chain/src/chain.rs | 1 + flexidag/src/blockdag.rs | 20 +++++++++++++------- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/chain/src/chain.rs b/chain/src/chain.rs index acddd4de70..92875a0a0c 100644 --- a/chain/src/chain.rs +++ b/chain/src/chain.rs @@ -375,6 +375,7 @@ impl BlockChain { )?; let excluded_txns = opened_block.push_txns(user_txns)?; let template = opened_block.finalize()?; + Ok((template, excluded_txns)) } diff --git a/flexidag/src/blockdag.rs b/flexidag/src/blockdag.rs index 885bc4dc08..7408e59250 100644 --- a/flexidag/src/blockdag.rs +++ b/flexidag/src/blockdag.rs @@ -452,11 +452,6 @@ impl BlockDAG { // Generate ghostdag data let parents = header.parents(); - debug!( - "start to get the ghost data from block: {:?}, number: {:?}", - header.id(), - header.number() - ); let ghostdata = match self.ghostdata_by_hash(header.id())? { None => { // It must be the dag genesis if header is a format for a single chain @@ -766,13 +761,19 @@ impl BlockDAG { self.pruning_point_manager().reachability_service() } + // return true the block processing will be going into the single chain logic, + // which means that this is a historical block that converge to the next pruning point + // return false it will be going into the ghost logic, + // which means that this is a new block that will be added by the ghost protocol that enhance the parallelism of the block processing. + // for vega, the special situation is: + // the pruning logic was delivered after vega running for a long time, so the historical block will be processed by the single chain logic. fn check_historical_block( &self, header: &BlockHeader, latest_pruning_point: Option, ) -> Result { if let Some(pruning_point) = latest_pruning_point { - if pruning_point == HashValue::zero() { + if pruning_point == HashValue::zero() && header.chain_id().is_vega() { info!("pruning point is zero"); Ok(true) } else if header.pruning_point() == pruning_point { @@ -781,6 +782,11 @@ impl BlockDAG { pruning_point ); Ok(false) + } else if header.pruning_point() != HashValue::zero() + && pruning_point == HashValue::zero() + { + info!("pruning point is not zero, but the latest pruning point is zero, pruning point: {:?}, latest pruning point: {:?}, this is the first pruning point", header.pruning_point(), pruning_point); + Ok(false) } else if self.check_ancestor_of(header.pruning_point(), pruning_point)? { info!("pruning point is the ancestor of the latest pruning point, pruning point: {:?}, latest pruning point: {:?}", header.pruning_point(), pruning_point); Ok(true) @@ -789,7 +795,7 @@ impl BlockDAG { Ok(false) } } else { - Ok(true) + Ok(false) } } From 8d3e30c4b171302bea019ffad28e9a806b9bdb1f Mon Sep 17 00:00:00 2001 From: jackzhhuang Date: Thu, 16 Jan 2025 15:17:27 +0800 Subject: [PATCH 17/23] zero hash is genesis so it is the ancestor of every block --- flexidag/src/blockdag.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flexidag/src/blockdag.rs b/flexidag/src/blockdag.rs index 7408e59250..b12d5446f5 100644 --- a/flexidag/src/blockdag.rs +++ b/flexidag/src/blockdag.rs @@ -205,6 +205,9 @@ impl BlockDAG { } pub fn check_ancestor_of(&self, ancestor: Hash, descendant: Hash) -> anyhow::Result { + if ancestor == Hash::zero() { + return Ok(true); + } inquirer::is_dag_ancestor_of( &*self.storage.reachability_store.read(), ancestor, From adf48bed674c7886d05e55b28edcc75831024610 Mon Sep 17 00:00:00 2001 From: jackzhhuang Date: Thu, 16 Jan 2025 15:57:55 +0800 Subject: [PATCH 18/23] fix checking ancestor in pruning manager --- flexidag/src/prune/pruning_point_manager.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flexidag/src/prune/pruning_point_manager.rs b/flexidag/src/prune/pruning_point_manager.rs index a81597cf69..8ccf4e862f 100644 --- a/flexidag/src/prune/pruning_point_manager.rs +++ b/flexidag/src/prune/pruning_point_manager.rs @@ -42,6 +42,9 @@ impl PruningPointManagerT { current_pruning_point: HashValue, next_pruning_point: HashValue, ) -> anyhow::Result> { + if next_pruning_point == HashValue::zero() { + return Ok(dag_state.tips.clone()); + } if current_pruning_point == next_pruning_point { return Ok(dag_state.tips.clone()); } From 717dc8aa195e83c6a49285024f95eeefde8e70a6 Mon Sep 17 00:00:00 2001 From: jackzhhuang Date: Fri, 17 Jan 2025 14:55:43 +0800 Subject: [PATCH 19/23] add clearly logic --- flexidag/src/blockdag.rs | 42 +++++++++++++-------- flexidag/src/prune/pruning_point_manager.rs | 18 ++------- 2 files changed, 31 insertions(+), 29 deletions(-) diff --git a/flexidag/src/blockdag.rs b/flexidag/src/blockdag.rs index b12d5446f5..533607b82e 100644 --- a/flexidag/src/blockdag.rs +++ b/flexidag/src/blockdag.rs @@ -720,7 +720,7 @@ impl BlockDAG { let pruned_ghostdata = self.ghost_dag_manager().ghostdag(&pruned_tips)?; let mergeset_blues = pruned_ghostdata.mergeset_blues.as_ref().clone(); info!( - "the pruning was triggered, previous tips: {:?}, the current tips: {:?}, the next pruning point is: {:?}, the current ghostdata's selected parent: {:?}, blue blocks are: {:?} and its red blocks are: {:?}", + "the pruning was triggered, before pruning, the tips: {:?}, after pruning tips: {:?}, the next pruning point is: {:?}, the current ghostdata's selected parent: {:?}, blue blocks are: {:?} and its red blocks are: {:?}", pruned_tips, dag_state.tips, next_pruning_point, pruned_ghostdata.selected_parent, pruned_ghostdata.mergeset_blues, pruned_ghostdata.mergeset_reds.len(), ); anyhow::Ok(MineNewDagBlockInfo { @@ -776,25 +776,24 @@ impl BlockDAG { latest_pruning_point: Option, ) -> Result { if let Some(pruning_point) = latest_pruning_point { - if pruning_point == HashValue::zero() && header.chain_id().is_vega() { - info!("pruning point is zero"); - Ok(true) + if header.chain_id().is_vega() { + match (header.pruning_point() == HashValue::zero(), pruning_point == HashValue::zero()) { + (true, true) => Ok(true), + (true, false) => bail!("the block header pruning point is zero, but the latest pruning point is not zero"), + (false, true) => Ok(false), + (false, false) => { + if self.check_ancestor_of(header.pruning_point(), pruning_point)? { + Ok(true) + } else { + Ok(false) + } + } + } } else if header.pruning_point() == pruning_point { - info!( - "pruning point is the same as the latest pruning point, pruning point: {:?}", - pruning_point - ); - Ok(false) - } else if header.pruning_point() != HashValue::zero() - && pruning_point == HashValue::zero() - { - info!("pruning point is not zero, but the latest pruning point is zero, pruning point: {:?}, latest pruning point: {:?}, this is the first pruning point", header.pruning_point(), pruning_point); Ok(false) } else if self.check_ancestor_of(header.pruning_point(), pruning_point)? { - info!("pruning point is the ancestor of the latest pruning point, pruning point: {:?}, latest pruning point: {:?}", header.pruning_point(), pruning_point); Ok(true) } else { - info!("pruning point is not the ancestor of the latest pruning point, pruning point: {:?}, latest pruning point: {:?}", header.pruning_point(), pruning_point); Ok(false) } } else { @@ -808,10 +807,23 @@ impl BlockDAG { header: &BlockHeader, latest_pruning_point: Option, ) -> Result { + info!( + "checking historical block: header pruning point: {:?}, latest pruning point: {:?}", + header.pruning_point(), + latest_pruning_point + ); if self.check_historical_block(header, latest_pruning_point)? { + info!( + "the block is a historical block, the header id: {:?}", + header.id() + ); self.ghost_dag_manager() .verify_and_ghostdata(blue_blocks, header) } else { + info!( + "the block is not a historical block, the header id: {:?}", + header.id() + ); self.ghost_dag_manager().ghostdag(&header.parents()) } } diff --git a/flexidag/src/prune/pruning_point_manager.rs b/flexidag/src/prune/pruning_point_manager.rs index 8ccf4e862f..b1b41f7f2b 100644 --- a/flexidag/src/prune/pruning_point_manager.rs +++ b/flexidag/src/prune/pruning_point_manager.rs @@ -69,20 +69,14 @@ impl PruningPointManagerT { pruning_depth: u64, pruning_finality: u64, ) -> anyhow::Result { + info!( + "previous_pruning_point: {:?}, previous_ghostdata: {:?}, next_ghostdata: {:?}, pruning_depth: {}, pruning_finality: {}", + previous_pruning_point, previous_ghostdata.to_compact(), next_ghostdata.to_compact(), pruning_depth, pruning_finality + ); let min_required_blue_score_for_next_pruning_point = (self.finality_score(previous_ghostdata.blue_score, pruning_finality) + 1) * pruning_finality; - debug!( - "min_required_blue_score_for_next_pruning_point: {:?}", - min_required_blue_score_for_next_pruning_point - ); - - debug!("previous_pruning_point: {:?}, previous_ghostdata: {:?}, next_ghostdata: {:?}, pruning_depth: {:?}, pruning_finality: {:?}", - previous_pruning_point, previous_ghostdata, next_ghostdata, - pruning_depth, pruning_finality, - ); - let mut latest_pruning_ghost_data = previous_ghostdata.to_compact(); if min_required_blue_score_for_next_pruning_point + pruning_depth <= next_ghostdata.blue_score @@ -93,10 +87,6 @@ impl PruningPointManagerT { true, ) { let next_pruning_ghostdata = self.ghost_dag_store.get_data(child)?; - debug!( - "child: {:?}, observer2.blue_score: {:?}, next_pruning_ghostdata.blue_score: {:?}", - child, next_ghostdata.blue_score, next_pruning_ghostdata.blue_score - ); if next_ghostdata.blue_score - next_pruning_ghostdata.blue_score < pruning_depth { break; } From e0aeb55bc022725e890e75d8bc3d268b3081fec1 Mon Sep 17 00:00:00 2001 From: jackzhhuang Date: Fri, 17 Jan 2025 15:01:54 +0800 Subject: [PATCH 20/23] fix fmt --- flexidag/src/prune/pruning_point_manager.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexidag/src/prune/pruning_point_manager.rs b/flexidag/src/prune/pruning_point_manager.rs index b1b41f7f2b..aa1c6ac9dc 100644 --- a/flexidag/src/prune/pruning_point_manager.rs +++ b/flexidag/src/prune/pruning_point_manager.rs @@ -1,5 +1,5 @@ use starcoin_crypto::HashValue; -use starcoin_logger::prelude::{debug, info}; +use starcoin_logger::prelude::info; use crate::reachability::reachability_service::ReachabilityService; use crate::{ From 7e8ef5f1a55a65d2fbbef598cb1bbf0743a3328b Mon Sep 17 00:00:00 2001 From: jackzhhuang Date: Fri, 17 Jan 2025 16:09:30 +0800 Subject: [PATCH 21/23] return previous pruning point --- flexidag/src/prune/pruning_point_manager.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/flexidag/src/prune/pruning_point_manager.rs b/flexidag/src/prune/pruning_point_manager.rs index aa1c6ac9dc..8cd91ad397 100644 --- a/flexidag/src/prune/pruning_point_manager.rs +++ b/flexidag/src/prune/pruning_point_manager.rs @@ -107,7 +107,12 @@ impl PruningPointManagerT { if latest_pruning_ghost_data.selected_parent == previous_ghostdata.to_compact().selected_parent { - anyhow::Ok(HashValue::zero()) // still genesis + // anyhow::Ok(HashValue::zero()) // still genesis + if previous_pruning_point == HashValue::zero() { + anyhow::Ok(HashValue::zero()) + } else { + anyhow::Ok(previous_ghostdata.to_compact().selected_parent) + } } else { anyhow::Ok(latest_pruning_ghost_data.selected_parent) } From f2e126bfdbc72643c33de5b527e43da5c629b403 Mon Sep 17 00:00:00 2001 From: jackzhhuang Date: Fri, 17 Jan 2025 22:02:08 +0800 Subject: [PATCH 22/23] fix the check historical blocks --- flexidag/src/blockdag.rs | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/flexidag/src/blockdag.rs b/flexidag/src/blockdag.rs index 533607b82e..aea4356888 100644 --- a/flexidag/src/blockdag.rs +++ b/flexidag/src/blockdag.rs @@ -776,25 +776,26 @@ impl BlockDAG { latest_pruning_point: Option, ) -> Result { if let Some(pruning_point) = latest_pruning_point { - if header.chain_id().is_vega() { - match (header.pruning_point() == HashValue::zero(), pruning_point == HashValue::zero()) { - (true, true) => Ok(true), - (true, false) => bail!("the block header pruning point is zero, but the latest pruning point is not zero"), - (false, true) => Ok(false), - (false, false) => { - if self.check_ancestor_of(header.pruning_point(), pruning_point)? { - Ok(true) - } else { - Ok(false) - } + match ( + header.pruning_point() == HashValue::zero(), + pruning_point == HashValue::zero(), + ) { + (true, true) => { + if header.chain_id().is_vega() { + Ok(true) + } else { + Ok(false) + } + } + (true, false) => Ok(true), + (false, true) => Ok(false), + (false, false) => { + if self.check_ancestor_of(header.pruning_point(), pruning_point)? { + Ok(true) + } else { + Ok(false) } } - } else if header.pruning_point() == pruning_point { - Ok(false) - } else if self.check_ancestor_of(header.pruning_point(), pruning_point)? { - Ok(true) - } else { - Ok(false) } } else { Ok(false) From 323e42ceefbd6b60e386f6d9f4d0e9a16084133e Mon Sep 17 00:00:00 2001 From: jackzhhuang Date: Tue, 28 Jan 2025 00:22:50 +0800 Subject: [PATCH 23/23] check the same pruning point when checking the historical blocks --- flexidag/src/blockdag.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/flexidag/src/blockdag.rs b/flexidag/src/blockdag.rs index aea4356888..a783118447 100644 --- a/flexidag/src/blockdag.rs +++ b/flexidag/src/blockdag.rs @@ -790,7 +790,9 @@ impl BlockDAG { (true, false) => Ok(true), (false, true) => Ok(false), (false, false) => { - if self.check_ancestor_of(header.pruning_point(), pruning_point)? { + if header.pruning_point() == pruning_point { + Ok(false) + } else if self.check_ancestor_of(header.pruning_point(), pruning_point)? { Ok(true) } else { Ok(false)