From 9a18abcffe2b1ecbec466b1fc3eb435492240588 Mon Sep 17 00:00:00 2001 From: EthanYuan Date: Wed, 22 Nov 2023 21:34:16 +0800 Subject: [PATCH 1/2] indexer support setting init tip. --- resource/ckb.toml | 8 ++++++++ util/app-config/src/configs/indexer.rs | 9 +++++++++ util/indexer/src/service.rs | 19 ++++++++++++++++++- 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/resource/ckb.toml b/resource/ckb.toml index 6c60901b97..94629a49ff 100644 --- a/resource/ckb.toml +++ b/resource/ckb.toml @@ -199,3 +199,11 @@ block_uncles_cache_size = 30 # [indexer_v2] # # Indexing the pending txs in the ckb tx-pool # index_tx_pool = false +# # Customize block filtering rules to index only retained blocks +# block_filter = "block.header.number.to_uint() >= \"0x0\".to_uint()" +# # Customize cell filtering rules to index only retained cells +# cell_filter = "let script = output.type;script!=() && script.code_hash == \"0x00000000000000000000000000000000000000000000000000545950455f4944\"" +# # The initial tip number and hash can be set as the starting height for building indexes in indexer-r. Effective only during the initial index creation. +# init_tip_number = 10000 +# # The initial tip hash must match the tip number; otherwise, it will result in a rollback. +# init_tip_hash = "0x8fbd0ec887159d2814cee475911600e3589849670f5ee1ed9798b38fdeef4e44" diff --git a/util/app-config/src/configs/indexer.rs b/util/app-config/src/configs/indexer.rs index e1621dfad0..c83fae2385 100644 --- a/util/app-config/src/configs/indexer.rs +++ b/util/app-config/src/configs/indexer.rs @@ -1,3 +1,4 @@ +use ckb_types::H256; use serde::{Deserialize, Serialize}; use std::num::NonZeroUsize; use std::path::{Path, PathBuf}; @@ -28,6 +29,12 @@ pub struct IndexerConfig { /// Maximal db info log files to be kept. #[serde(default)] pub db_keep_log_file_num: Option, + /// The init tip block number + #[serde(default)] + pub init_tip_number: Option, + /// The init tip block hash + #[serde(default)] + pub init_tip_hash: Option, } const fn default_poll_interval() -> u64 { @@ -45,6 +52,8 @@ impl Default for IndexerConfig { cell_filter: None, db_background_jobs: None, db_keep_log_file_num: None, + init_tip_number: None, + init_tip_hash: None, } } } diff --git a/util/indexer/src/service.rs b/util/indexer/src/service.rs index 1526074dd5..6a869f89b7 100644 --- a/util/indexer/src/service.rs +++ b/util/indexer/src/service.rs @@ -2,7 +2,7 @@ use crate::indexer::{self, extract_raw_data, CustomFilters, Indexer, Key, KeyPrefix, Value}; use crate::pool::Pool; -use crate::store::{IteratorDirection, RocksdbStore, SecondaryDB, Store}; +use crate::store::{Batch, IteratorDirection, RocksdbStore, SecondaryDB, Store}; use crate::error::Error; use ckb_app_config::{DBConfig, IndexerConfig}; @@ -46,8 +46,10 @@ pub struct IndexerService { impl IndexerService { /// Construct new Indexer service instance from DBConfig and IndexerConfig pub fn new(ckb_db_config: &DBConfig, config: &IndexerConfig, async_handle: Handle) -> Self { + let is_store_initialized = config.store.exists(); let store_opts = Self::indexer_store_options(config); let store = RocksdbStore::new(&store_opts, &config.store); + Self::apply_init_tip(is_store_initialized, config, &store); let pool = if config.index_tx_pool { Some(Arc::new(RwLock::new(Pool::default()))) } else { @@ -247,6 +249,21 @@ impl IndexerService { ); opts } + + fn apply_init_tip(is_store_initialized: bool, config: &IndexerConfig, store: &RocksdbStore) { + if !is_store_initialized + && config.init_tip_hash.is_some() + && config.init_tip_number.is_some() + { + let block_number = config.init_tip_number.unwrap(); + let block_hash = config.init_tip_hash.as_ref().unwrap(); + let mut batch = store.batch().expect("create batch should be OK"); + batch + .put_kv(Key::Header(block_number, &block_hash.pack(), true), vec![]) + .expect("insert init tip header should be OK"); + batch.commit().expect("commit batch should be OK"); + } + } } /// Handle to the indexer. From 1a60bdbd374031fb04113b305b3bfcda8c7afd0a Mon Sep 17 00:00:00 2001 From: EthanYuan Date: Wed, 22 Nov 2023 23:32:28 +0800 Subject: [PATCH 2/2] add tests for setting init tip: - get_indexer_tip - rollback --- resource/ckb.toml | 2 +- util/indexer/src/indexer.rs | 32 +++++++++++++++++++++++++++- util/indexer/src/service.rs | 42 ++++++++++++++++++++++++++++++------- 3 files changed, 66 insertions(+), 10 deletions(-) diff --git a/resource/ckb.toml b/resource/ckb.toml index 94629a49ff..aa09a297a1 100644 --- a/resource/ckb.toml +++ b/resource/ckb.toml @@ -205,5 +205,5 @@ block_uncles_cache_size = 30 # cell_filter = "let script = output.type;script!=() && script.code_hash == \"0x00000000000000000000000000000000000000000000000000545950455f4944\"" # # The initial tip number and hash can be set as the starting height for building indexes in indexer-r. Effective only during the initial index creation. # init_tip_number = 10000 -# # The initial tip hash must match the tip number; otherwise, it will result in a rollback. +# # The initial tip hash must match the tip number; otherwise, it will result in a rollback to empty. # init_tip_hash = "0x8fbd0ec887159d2814cee475911600e3589849670f5ee1ed9798b38fdeef4e44" diff --git a/util/indexer/src/indexer.rs b/util/indexer/src/indexer.rs index 45939f708d..3ecc706527 100644 --- a/util/indexer/src/indexer.rs +++ b/util/indexer/src/indexer.rs @@ -997,7 +997,7 @@ impl CustomFilters { #[cfg(test)] mod tests { use super::*; - use crate::store::RocksdbStore; + use crate::{store::RocksdbStore, IndexerService}; use ckb_types::{ bytes::Bytes, core::{ @@ -2015,6 +2015,20 @@ mod tests { ) } + fn new_indexer_with_init_tip( + prefix: &str, + init_tip_number: Option, + init_tip_hash: Option, + ) -> Indexer { + let tmp_dir = tempfile::Builder::new().prefix(prefix).tempdir().unwrap(); + let store = RocksdbStore::new( + &RocksdbStore::default_options(), + tmp_dir.path().to_str().unwrap(), + ); + IndexerService::apply_init_tip(false, init_tip_number, &init_tip_hash, &store); + Indexer::new(store, 10, 1, None, CustomFilters::new(None, None)) + } + #[test] fn with_custom_block_filter() { let indexer = new_indexer_with_custom_filters::( @@ -2364,4 +2378,20 @@ mod tests { .len() ); } + + #[test] + fn rollback_with_set_init_tip() { + let indexer = new_indexer_with_init_tip( + "rollback_with_set_init_tip", + Some(1000), + Some(H256::default()), + ); + + indexer.rollback().unwrap(); + + // tip should be None and store should be empty; + assert!(indexer.tip().unwrap().is_none()); + let mut iter = indexer.store.iter([], IteratorDirection::Forward).unwrap(); + assert!(iter.next().is_none()); + } } diff --git a/util/indexer/src/service.rs b/util/indexer/src/service.rs index 6a869f89b7..6c7374beb6 100644 --- a/util/indexer/src/service.rs +++ b/util/indexer/src/service.rs @@ -49,7 +49,12 @@ impl IndexerService { let is_store_initialized = config.store.exists(); let store_opts = Self::indexer_store_options(config); let store = RocksdbStore::new(&store_opts, &config.store); - Self::apply_init_tip(is_store_initialized, config, &store); + Self::apply_init_tip( + is_store_initialized, + config.init_tip_number, + &config.init_tip_hash, + &store, + ); let pool = if config.index_tx_pool { Some(Arc::new(RwLock::new(Pool::default()))) } else { @@ -250,16 +255,21 @@ impl IndexerService { opts } - fn apply_init_tip(is_store_initialized: bool, config: &IndexerConfig, store: &RocksdbStore) { - if !is_store_initialized - && config.init_tip_hash.is_some() - && config.init_tip_number.is_some() + pub(crate) fn apply_init_tip( + is_store_initialized: bool, + init_tip_number: Option, + init_tip_hash: &Option, + store: &RocksdbStore, + ) { + if let (true, Some(init_tip_number), Some(init_tip_hash)) = + (!is_store_initialized, init_tip_number, init_tip_hash) { - let block_number = config.init_tip_number.unwrap(); - let block_hash = config.init_tip_hash.as_ref().unwrap(); let mut batch = store.batch().expect("create batch should be OK"); batch - .put_kv(Key::Header(block_number, &block_hash.pack(), true), vec![]) + .put_kv( + Key::Header(init_tip_number, &init_tip_hash.pack(), true), + vec![], + ) .expect("insert init tip header should be OK"); batch.commit().expect("commit batch should be OK"); } @@ -1571,6 +1581,22 @@ mod tests { ); } + #[test] + fn rpc_get_indexer_tip_with_set_init_tip() { + let store = new_store("rpc_get_indexer_tip_with_set_init_tip"); + IndexerService::apply_init_tip(false, Some(10000), &Some(H256::default()), &store); + let pool = Arc::new(RwLock::new(Pool::default())); + let rpc = IndexerHandle { + store, + pool: Some(Arc::clone(&pool)), + }; + + // test get_tip rpc + let tip = rpc.get_indexer_tip().unwrap().unwrap(); + assert_eq!(H256::default(), tip.block_hash); + assert_eq!(10000, tip.block_number.value()); + } + #[test] fn script_search_mode_rpc() { let store = new_store("script_search_mode_rpc");