From 89bbaac4f14ae84185c7eac636a886e8bef6a25e Mon Sep 17 00:00:00 2001 From: Rafael Cardenas Date: Mon, 9 Sep 2024 20:35:01 -0600 Subject: [PATCH 1/4] fix: add a max timeout for outgoing requests --- components/chainhook-sdk/src/chainhooks/bitcoin/mod.rs | 3 ++- components/chainhook-sdk/src/chainhooks/stacks/mod.rs | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/components/chainhook-sdk/src/chainhooks/bitcoin/mod.rs b/components/chainhook-sdk/src/chainhooks/bitcoin/mod.rs index 7d49ec127..633f310ad 100644 --- a/components/chainhook-sdk/src/chainhooks/bitcoin/mod.rs +++ b/components/chainhook-sdk/src/chainhooks/bitcoin/mod.rs @@ -21,7 +21,7 @@ use serde::{de, Deserialize, Deserializer}; use serde_json::Value as JsonValue; use std::{ collections::{BTreeMap, HashMap, HashSet}, - str::FromStr, + str::FromStr, time::Duration, }; use reqwest::RequestBuilder; @@ -766,6 +766,7 @@ pub fn handle_bitcoin_hook_action<'a>( match &trigger.chainhook.action { HookAction::HttpPost(http) => { let client = Client::builder() + .timeout(Duration::from_secs(30)) .build() .map_err(|e| format!("unable to build http client: {}", e.to_string()))?; let host = format!("{}", http.url); diff --git a/components/chainhook-sdk/src/chainhooks/stacks/mod.rs b/components/chainhook-sdk/src/chainhooks/stacks/mod.rs index 6bad60849..b3fe0334f 100644 --- a/components/chainhook-sdk/src/chainhooks/stacks/mod.rs +++ b/components/chainhook-sdk/src/chainhooks/stacks/mod.rs @@ -22,6 +22,7 @@ use schemars::JsonSchema; use serde_json::Value as JsonValue; use std::collections::{BTreeMap, HashMap}; use std::io::Cursor; +use std::time::Duration; use reqwest::RequestBuilder; @@ -1337,6 +1338,7 @@ pub fn handle_stacks_hook_action<'a>( match &trigger.chainhook.action { HookAction::HttpPost(http) => { let client = Client::builder() + .timeout(Duration::from_secs(30)) .build() .map_err(|e| format!("unable to build http client: {}", e.to_string()))?; let host = format!("{}", http.url); From 3d1d9a849affed137721db12142ea74a2a275a24 Mon Sep 17 00:00:00 2001 From: Rafael Cardenas Date: Wed, 9 Oct 2024 09:16:20 -0600 Subject: [PATCH 2/4] chore: start adding config --- components/chainhook-cli/src/config/file.rs | 6 ++++++ components/chainhook-cli/src/config/mod.rs | 18 ++++++++++++++++++ components/chainhook-cli/src/scan/bitcoin.rs | 2 +- components/chainhook-cli/src/scan/stacks.rs | 4 ++-- .../src/service/tests/helpers/mock_service.rs | 4 ++-- .../src/service/tests/observer_tests.rs | 1 + .../src/chainhooks/bitcoin/mod.rs | 10 +++++++--- .../chainhook-sdk/src/chainhooks/stacks/mod.rs | 9 +++++++-- components/chainhook-sdk/src/observer/mod.rs | 8 ++++++-- .../chainhook-sdk/src/observer/tests/mod.rs | 1 + 10 files changed, 51 insertions(+), 12 deletions(-) diff --git a/components/chainhook-cli/src/config/file.rs b/components/chainhook-cli/src/config/file.rs index 6ef2fa2fa..25e4abedb 100644 --- a/components/chainhook-cli/src/config/file.rs +++ b/components/chainhook-cli/src/config/file.rs @@ -5,6 +5,7 @@ pub struct ConfigFile { pub storage: StorageConfigFile, pub pox_config: Option, pub http_api: Option, + pub predicates: Option, pub event_source: Option>, pub limits: LimitsConfigFile, pub network: NetworkConfigFile, @@ -64,6 +65,11 @@ pub struct NetworkConfigFile { pub stacks_events_ingestion_port: Option, } +#[derive(Deserialize, Debug, Clone)] +pub struct PredicatesConfigFile { + pub payload_http_request_timeout_ms: Option, +} + #[derive(Deserialize, Debug, Clone)] #[serde(rename_all = "snake_case")] pub enum NetworkConfigMode { diff --git a/components/chainhook-cli/src/config/mod.rs b/components/chainhook-cli/src/config/mod.rs index fb2b6b2a3..0ee9ee588 100644 --- a/components/chainhook-cli/src/config/mod.rs +++ b/components/chainhook-cli/src/config/mod.rs @@ -30,6 +30,7 @@ pub struct Config { pub storage: StorageConfig, pub pox_config: PoxConfig, pub http_api: PredicatesApi, + pub predicates: PredicatesConfig, pub event_sources: Vec, pub limits: LimitsConfig, pub network: IndexerConfig, @@ -83,6 +84,11 @@ pub struct LimitsConfig { pub max_caching_memory_size_mb: usize, } +#[derive(Clone, Debug, PartialEq)] +pub struct PredicatesConfig { + pub payload_http_request_timeout_ms: Option, +} + #[derive(Clone, Debug, PartialEq)] pub struct MonitoringConfig { pub prometheus_monitoring_port: Option, @@ -117,6 +123,7 @@ impl Config { EventObserverConfig { bitcoin_rpc_proxy_enabled: true, registered_chainhooks: ChainhookStore::new(), + predicate_payload_http_request_timeout_ms: self.predicates.payload_http_request_timeout_ms, bitcoind_rpc_username: self.network.bitcoind_rpc_username.clone(), bitcoind_rpc_password: self.network.bitcoind_rpc_password.clone(), bitcoind_rpc_url: self.network.bitcoind_rpc_url.clone(), @@ -193,6 +200,14 @@ impl Config { }), }, }, + predicates: match config_file.predicates { + None => PredicatesConfig { + payload_http_request_timeout_ms: None, + }, + Some(predicates) => PredicatesConfig { + payload_http_request_timeout_ms: predicates.payload_http_request_timeout_ms, + }, + }, event_sources, limits: LimitsConfig { max_number_of_stacks_predicates: config_file @@ -357,6 +372,7 @@ impl Config { }, pox_config: PoxConfig::devnet_default(), http_api: PredicatesApi::Off, + predicates: PredicatesConfig { payload_http_request_timeout_ms: None }, event_sources: vec![], limits: LimitsConfig { max_number_of_bitcoin_predicates: BITCOIN_MAX_PREDICATE_REGISTRATION, @@ -390,6 +406,7 @@ impl Config { }, pox_config: PoxConfig::testnet_default(), http_api: PredicatesApi::Off, + predicates: PredicatesConfig { payload_http_request_timeout_ms: None }, event_sources: vec![EventSourceConfig::StacksTsvUrl(UrlConfig { file_url: DEFAULT_TESTNET_STACKS_TSV_ARCHIVE.into(), })], @@ -425,6 +442,7 @@ impl Config { }, pox_config: PoxConfig::mainnet_default(), http_api: PredicatesApi::Off, + predicates: PredicatesConfig { payload_http_request_timeout_ms: None }, event_sources: vec![EventSourceConfig::StacksTsvUrl(UrlConfig { file_url: DEFAULT_MAINNET_STACKS_TSV_ARCHIVE.into(), })], diff --git a/components/chainhook-cli/src/scan/bitcoin.rs b/components/chainhook-cli/src/scan/bitcoin.rs index e57d2ddfc..3427ef60b 100644 --- a/components/chainhook-cli/src/scan/bitcoin.rs +++ b/components/chainhook-cli/src/scan/bitcoin.rs @@ -290,7 +290,7 @@ pub async fn execute_predicates_action<'a>( gather_proofs(&trigger, &mut proofs, config, ctx); } let predicate_uuid = &trigger.chainhook.uuid; - match handle_bitcoin_hook_action(trigger, &proofs) { + match handle_bitcoin_hook_action(trigger, &proofs, &config) { Err(e) => { warn!( ctx.expect_logger(), diff --git a/components/chainhook-cli/src/scan/stacks.rs b/components/chainhook-cli/src/scan/stacks.rs index 9ba580399..2fd977cdb 100644 --- a/components/chainhook-cli/src/scan/stacks.rs +++ b/components/chainhook-cli/src/scan/stacks.rs @@ -338,7 +338,7 @@ pub async fn scan_stacks_chainstate_via_rocksdb_using_predicate( apply: hits_per_blocks, rollback: vec![], }; - let res = match handle_stacks_hook_action(trigger, &proofs, ctx) { + let res = match handle_stacks_hook_action(trigger, &proofs, &config.get_event_observer_config(), ctx) { Err(e) => { warn!( ctx.expect_logger(), @@ -525,7 +525,7 @@ pub async fn scan_stacks_chainstate_via_csv_using_predicate( apply: hits_per_blocks, rollback: vec![], }; - match handle_stacks_hook_action(trigger, &proofs, ctx) { + match handle_stacks_hook_action(trigger, &proofs, &config.get_event_observer_config(), ctx) { Err(e) => { error!(ctx.expect_logger(), "unable to handle action {}", e); } diff --git a/components/chainhook-cli/src/service/tests/helpers/mock_service.rs b/components/chainhook-cli/src/service/tests/helpers/mock_service.rs index b9c821429..42517b151 100644 --- a/components/chainhook-cli/src/service/tests/helpers/mock_service.rs +++ b/components/chainhook-cli/src/service/tests/helpers/mock_service.rs @@ -1,6 +1,5 @@ use crate::config::{ - Config, EventSourceConfig, LimitsConfig, MonitoringConfig, PathConfig, PredicatesApi, - PredicatesApiConfig, StorageConfig, DEFAULT_REDIS_URI, + Config, EventSourceConfig, LimitsConfig, MonitoringConfig, PathConfig, PredicatesApi, PredicatesApiConfig, PredicatesConfig, StorageConfig, DEFAULT_REDIS_URI }; use crate::scan::stacks::consolidate_local_stacks_chainstate_using_csv; use crate::service::{ @@ -293,6 +292,7 @@ pub fn get_chainhook_config( }; Config { http_api: PredicatesApi::On(api_config), + predicates: PredicatesConfig { payload_http_request_timeout_ms: None }, pox_config: PoxConfig::devnet_default(), storage: StorageConfig { working_dir: working_dir.into(), diff --git a/components/chainhook-cli/src/service/tests/observer_tests.rs b/components/chainhook-cli/src/service/tests/observer_tests.rs index f3ebd2009..79b51cc64 100644 --- a/components/chainhook-cli/src/service/tests/observer_tests.rs +++ b/components/chainhook-cli/src/service/tests/observer_tests.rs @@ -190,6 +190,7 @@ async fn it_responds_200_for_unimplemented_endpoints( }); let config = EventObserverConfig { registered_chainhooks: ChainhookStore::new(), + predicate_payload_http_request_timeout_ms: None, bitcoin_rpc_proxy_enabled: false, bitcoind_rpc_username: String::new(), bitcoind_rpc_password: String::new(), diff --git a/components/chainhook-sdk/src/chainhooks/bitcoin/mod.rs b/components/chainhook-sdk/src/chainhooks/bitcoin/mod.rs index 6ee171b4f..e65c3add5 100644 --- a/components/chainhook-sdk/src/chainhooks/bitcoin/mod.rs +++ b/components/chainhook-sdk/src/chainhooks/bitcoin/mod.rs @@ -2,7 +2,7 @@ use super::types::{ append_error_context, validate_txid, ChainhookInstance, ExactMatchingRule, HookAction, MatchingRule, PoxConfig, TxinPredicate, }; -use crate::utils::{Context, MAX_BLOCK_HEIGHTS_ENTRIES}; +use crate::{observer::EventObserverConfig, utils::{Context, MAX_BLOCK_HEIGHTS_ENTRIES}}; use bitcoincore_rpc_json::bitcoin::{address::Payload, Address}; use chainhook_types::{ @@ -760,11 +760,15 @@ pub fn serialize_bitcoin_transactions_to_json( pub fn handle_bitcoin_hook_action<'a>( trigger: BitcoinTriggerChainhook<'a>, proofs: &HashMap<&'a TransactionIdentifier, String>, + config: &EventObserverConfig, ) -> Result { match &trigger.chainhook.action { HookAction::HttpPost(http) => { - let client = Client::builder() - .timeout(Duration::from_secs(30)) + let mut client_builder = Client::builder(); + if let Some(timeout) = config.predicate_payload_http_request_timeout_ms { + client_builder = client_builder.timeout(Duration::from_millis(timeout)); + } + let client = client_builder .build() .map_err(|e| format!("unable to build http client: {}", e))?; let host = http.url.to_string(); diff --git a/components/chainhook-sdk/src/chainhooks/stacks/mod.rs b/components/chainhook-sdk/src/chainhooks/stacks/mod.rs index fe079f226..2b7723ea6 100644 --- a/components/chainhook-sdk/src/chainhooks/stacks/mod.rs +++ b/components/chainhook-sdk/src/chainhooks/stacks/mod.rs @@ -1,3 +1,4 @@ +use crate::observer::EventObserverConfig; use crate::utils::{AbstractStacksBlock, Context, MAX_BLOCK_HEIGHTS_ENTRIES}; use super::types::{ @@ -1326,12 +1327,16 @@ pub fn serialize_stacks_payload_to_json<'a>( pub fn handle_stacks_hook_action<'a>( trigger: StacksTriggerChainhook<'a>, proofs: &HashMap<&'a TransactionIdentifier, String>, + config: &EventObserverConfig, ctx: &Context, ) -> Result { match &trigger.chainhook.action { HookAction::HttpPost(http) => { - let client = Client::builder() - .timeout(Duration::from_secs(30)) + let mut client_builder = Client::builder(); + if let Some(timeout) = config.predicate_payload_http_request_timeout_ms { + client_builder = client_builder.timeout(Duration::from_millis(timeout)); + } + let client = client_builder .build() .map_err(|e| format!("unable to build http client: {}", e))?; let host = http.url.to_string(); diff --git a/components/chainhook-sdk/src/observer/mod.rs b/components/chainhook-sdk/src/observer/mod.rs index d762ec294..65ed619aa 100644 --- a/components/chainhook-sdk/src/observer/mod.rs +++ b/components/chainhook-sdk/src/observer/mod.rs @@ -69,6 +69,7 @@ pub enum DataHandlerEvent { #[derive(Debug, Clone)] pub struct EventObserverConfig { pub registered_chainhooks: ChainhookStore, + pub predicate_payload_http_request_timeout_ms: Option, pub bitcoin_rpc_proxy_enabled: bool, pub bitcoind_rpc_username: String, pub bitcoind_rpc_password: String, @@ -290,6 +291,7 @@ impl BitcoinEventObserverConfigBuilder { }; Ok(EventObserverConfig { registered_chainhooks: ChainhookStore::new(), + predicate_payload_http_request_timeout_ms: None, bitcoin_rpc_proxy_enabled: false, bitcoind_rpc_username: self .bitcoind_rpc_username @@ -320,6 +322,7 @@ impl EventObserverConfig { pub fn default() -> Self { EventObserverConfig { registered_chainhooks: ChainhookStore::new(), + predicate_payload_http_request_timeout_ms: None, bitcoin_rpc_proxy_enabled: false, bitcoind_rpc_username: "devnet".into(), bitcoind_rpc_password: "devnet".into(), @@ -403,6 +406,7 @@ impl EventObserverConfig { let config = EventObserverConfig { bitcoin_rpc_proxy_enabled: false, registered_chainhooks: ChainhookStore::new(), + predicate_payload_http_request_timeout_ms: None, bitcoind_rpc_username: overrides .and_then(|c| c.bitcoind_rpc_username.clone()) .unwrap_or_else(|| "devnet".to_string()), @@ -1479,7 +1483,7 @@ pub async fn start_observer_commands_handler( } for chainhook_to_trigger in chainhooks_to_trigger.into_iter() { let predicate_uuid = &chainhook_to_trigger.chainhook.uuid; - match handle_bitcoin_hook_action(chainhook_to_trigger, &proofs) { + match handle_bitcoin_hook_action(chainhook_to_trigger, &proofs, &config) { Err(e) => { // todo: we may want to set predicates that reach this branch as interrupted, // but for now we will error to see if this problem occurs. @@ -1668,7 +1672,7 @@ pub async fn start_observer_commands_handler( let proofs = HashMap::new(); for chainhook_to_trigger in chainhooks_to_trigger.into_iter() { let predicate_uuid = &chainhook_to_trigger.chainhook.uuid; - match handle_stacks_hook_action(chainhook_to_trigger, &proofs, &ctx) { + match handle_stacks_hook_action(chainhook_to_trigger, &proofs, &config, &ctx) { Err(e) => { ctx.try_log(|logger| { // todo: we may want to set predicates that reach this branch as interrupted, diff --git a/components/chainhook-sdk/src/observer/tests/mod.rs b/components/chainhook-sdk/src/observer/tests/mod.rs index 32b19893c..b14538331 100644 --- a/components/chainhook-sdk/src/observer/tests/mod.rs +++ b/components/chainhook-sdk/src/observer/tests/mod.rs @@ -40,6 +40,7 @@ use super::{ObserverEvent, DEFAULT_INGESTION_PORT}; fn generate_test_config() -> (EventObserverConfig, ChainhookStore) { let config: EventObserverConfig = EventObserverConfig { registered_chainhooks: ChainhookStore::new(), + predicate_payload_http_request_timeout_ms: None, bitcoin_rpc_proxy_enabled: false, bitcoind_rpc_username: "user".into(), bitcoind_rpc_password: "user".into(), From 971ee385b3001517d9b693f21896f564ef75c6d9 Mon Sep 17 00:00:00 2001 From: Rafael Cardenas Date: Wed, 9 Oct 2024 09:59:24 -0600 Subject: [PATCH 3/4] fix: rename to predicates config --- components/chainhook-cli/src/config/mod.rs | 23 ++++---- components/chainhook-cli/src/scan/bitcoin.rs | 2 +- components/chainhook-cli/src/scan/stacks.rs | 30 ++++++++--- .../src/service/tests/helpers/mock_service.rs | 30 +++++------ .../src/service/tests/observer_tests.rs | 4 +- .../src/chainhooks/bitcoin/mod.rs | 6 +-- .../src/chainhooks/stacks/mod.rs | 6 +-- .../chainhook-sdk/src/chainhooks/tests/mod.rs | 11 ++-- components/chainhook-sdk/src/observer/mod.rs | 53 +++++++++++++++---- .../chainhook-sdk/src/observer/tests/mod.rs | 3 +- 10 files changed, 111 insertions(+), 57 deletions(-) diff --git a/components/chainhook-cli/src/config/mod.rs b/components/chainhook-cli/src/config/mod.rs index 0ee9ee588..cba31806e 100644 --- a/components/chainhook-cli/src/config/mod.rs +++ b/components/chainhook-cli/src/config/mod.rs @@ -3,7 +3,7 @@ pub mod generator; use chainhook_sdk::chainhooks::types::{ChainhookStore, PoxConfig}; pub use chainhook_sdk::indexer::IndexerConfig; -use chainhook_sdk::observer::EventObserverConfig; +use chainhook_sdk::observer::{EventObserverConfig, PredicatesConfig}; use chainhook_sdk::types::{ BitcoinBlockSignaling, BitcoinNetwork, StacksNetwork, StacksNodeConfig, }; @@ -84,11 +84,6 @@ pub struct LimitsConfig { pub max_caching_memory_size_mb: usize, } -#[derive(Clone, Debug, PartialEq)] -pub struct PredicatesConfig { - pub payload_http_request_timeout_ms: Option, -} - #[derive(Clone, Debug, PartialEq)] pub struct MonitoringConfig { pub prometheus_monitoring_port: Option, @@ -123,7 +118,9 @@ impl Config { EventObserverConfig { bitcoin_rpc_proxy_enabled: true, registered_chainhooks: ChainhookStore::new(), - predicate_payload_http_request_timeout_ms: self.predicates.payload_http_request_timeout_ms, + predicates_config: PredicatesConfig { + payload_http_request_timeout_ms: self.predicates.payload_http_request_timeout_ms, + }, bitcoind_rpc_username: self.network.bitcoind_rpc_username.clone(), bitcoind_rpc_password: self.network.bitcoind_rpc_password.clone(), bitcoind_rpc_url: self.network.bitcoind_rpc_url.clone(), @@ -372,7 +369,9 @@ impl Config { }, pox_config: PoxConfig::devnet_default(), http_api: PredicatesApi::Off, - predicates: PredicatesConfig { payload_http_request_timeout_ms: None }, + predicates: PredicatesConfig { + payload_http_request_timeout_ms: None, + }, event_sources: vec![], limits: LimitsConfig { max_number_of_bitcoin_predicates: BITCOIN_MAX_PREDICATE_REGISTRATION, @@ -406,7 +405,9 @@ impl Config { }, pox_config: PoxConfig::testnet_default(), http_api: PredicatesApi::Off, - predicates: PredicatesConfig { payload_http_request_timeout_ms: None }, + predicates: PredicatesConfig { + payload_http_request_timeout_ms: None, + }, event_sources: vec![EventSourceConfig::StacksTsvUrl(UrlConfig { file_url: DEFAULT_TESTNET_STACKS_TSV_ARCHIVE.into(), })], @@ -442,7 +443,9 @@ impl Config { }, pox_config: PoxConfig::mainnet_default(), http_api: PredicatesApi::Off, - predicates: PredicatesConfig { payload_http_request_timeout_ms: None }, + predicates: PredicatesConfig { + payload_http_request_timeout_ms: None, + }, event_sources: vec![EventSourceConfig::StacksTsvUrl(UrlConfig { file_url: DEFAULT_MAINNET_STACKS_TSV_ARCHIVE.into(), })], diff --git a/components/chainhook-cli/src/scan/bitcoin.rs b/components/chainhook-cli/src/scan/bitcoin.rs index 3427ef60b..fef4e507e 100644 --- a/components/chainhook-cli/src/scan/bitcoin.rs +++ b/components/chainhook-cli/src/scan/bitcoin.rs @@ -290,7 +290,7 @@ pub async fn execute_predicates_action<'a>( gather_proofs(&trigger, &mut proofs, config, ctx); } let predicate_uuid = &trigger.chainhook.uuid; - match handle_bitcoin_hook_action(trigger, &proofs, &config) { + match handle_bitcoin_hook_action(trigger, &proofs, &config.predicates_config) { Err(e) => { warn!( ctx.expect_logger(), diff --git a/components/chainhook-cli/src/scan/stacks.rs b/components/chainhook-cli/src/scan/stacks.rs index 2fd977cdb..279775b74 100644 --- a/components/chainhook-cli/src/scan/stacks.rs +++ b/components/chainhook-cli/src/scan/stacks.rs @@ -97,9 +97,11 @@ pub async fn get_canonical_fork_from_tsv( for result in reader_builder.deserialize() { line += 1; let record: Record = result.unwrap(); - if let RecordKind::StacksBlockReceived = &record.kind { if let Err(_e) = record_tx.send(Some((record, line))) { - break; - } }; + if let RecordKind::StacksBlockReceived = &record.kind { + if let Err(_e) = record_tx.send(Some((record, line))) { + break; + } + }; } let _ = record_tx.send(None); }) @@ -338,7 +340,12 @@ pub async fn scan_stacks_chainstate_via_rocksdb_using_predicate( apply: hits_per_blocks, rollback: vec![], }; - let res = match handle_stacks_hook_action(trigger, &proofs, &config.get_event_observer_config(), ctx) { + let res = match handle_stacks_hook_action( + trigger, + &proofs, + &config.get_event_observer_config().predicates_config, + ctx, + ) { Err(e) => { warn!( ctx.expect_logger(), @@ -487,7 +494,9 @@ pub async fn scan_stacks_chainstate_via_csv_using_predicate( let mut tsv_line = String::new(); while tsv_current_line < tsv_line_number { tsv_line.clear(); - let bytes_read = tsv_reader.read_line(&mut tsv_line).map_err(|e| e.to_string())?; + let bytes_read = tsv_reader + .read_line(&mut tsv_line) + .map_err(|e| e.to_string())?; if bytes_read == 0 { return Err("Unexpected EOF when reading TSV".to_string()); } @@ -525,7 +534,12 @@ pub async fn scan_stacks_chainstate_via_csv_using_predicate( apply: hits_per_blocks, rollback: vec![], }; - match handle_stacks_hook_action(trigger, &proofs, &config.get_event_observer_config(), ctx) { + match handle_stacks_hook_action( + trigger, + &proofs, + &config.get_event_observer_config().predicates_config, + ctx, + ) { Err(e) => { error!(ctx.expect_logger(), "unable to handle action {}", e); } @@ -604,7 +618,9 @@ pub async fn consolidate_local_stacks_chainstate_using_csv( let mut tsv_line = String::new(); while tsv_current_line < tsv_line_number { tsv_line.clear(); - let bytes_read = tsv_reader.read_line(&mut tsv_line).map_err(|e| e.to_string())?; + let bytes_read = tsv_reader + .read_line(&mut tsv_line) + .map_err(|e| e.to_string())?; if bytes_read == 0 { return Err("Unexpected EOF when reading TSV".to_string()); } diff --git a/components/chainhook-cli/src/service/tests/helpers/mock_service.rs b/components/chainhook-cli/src/service/tests/helpers/mock_service.rs index 42517b151..ae1196def 100644 --- a/components/chainhook-cli/src/service/tests/helpers/mock_service.rs +++ b/components/chainhook-cli/src/service/tests/helpers/mock_service.rs @@ -1,5 +1,6 @@ use crate::config::{ - Config, EventSourceConfig, LimitsConfig, MonitoringConfig, PathConfig, PredicatesApi, PredicatesApiConfig, PredicatesConfig, StorageConfig, DEFAULT_REDIS_URI + Config, EventSourceConfig, LimitsConfig, MonitoringConfig, PathConfig, PredicatesApi, + PredicatesApiConfig, StorageConfig, DEFAULT_REDIS_URI, }; use crate::scan::stacks::consolidate_local_stacks_chainstate_using_csv; use crate::service::{ @@ -7,6 +8,7 @@ use crate::service::{ PredicateStatus, Service, }; use chainhook_sdk::chainhooks::types::PoxConfig; +use chainhook_sdk::observer::PredicatesConfig; use chainhook_sdk::{ chainhooks::stacks::StacksChainhookSpecificationNetworkMap, chainhooks::types::{ChainhookInstance, ChainhookSpecificationNetworkMap}, @@ -81,12 +83,12 @@ pub async fn filter_predicate_status_from_all_predicates( match matching_predicate { Some(predicate) => match predicate.get("status") { Some(status) => { - return serde_json::from_value(status.clone()).map_err(|e| { - format!("failed to parse status {}", e) - }); + return serde_json::from_value(status.clone()) + .map_err(|e| format!("failed to parse status {}", e)); } None => { - return Err("no status field on matching get predicates result".to_string()) + return Err("no status field on matching get predicates result" + .to_string()) } }, None => { @@ -97,7 +99,9 @@ pub async fn filter_predicate_status_from_all_predicates( } } None => { - return Err("failed to parse get predicate response's result field".to_string()) + return Err( + "failed to parse get predicate response's result field".to_string() + ) } }, None => { @@ -266,10 +270,7 @@ pub fn flush_redis(port: u16) { let client = redis::Client::open(format!("redis://localhost:{port}/")) .expect("unable to connect to redis"); let mut predicate_db_conn = client.get_connection().expect("unable to connect to redis"); - let db_keys: Vec = predicate_db_conn - .scan_match("*") - .unwrap() - .collect(); + let db_keys: Vec = predicate_db_conn.scan_match("*").unwrap().collect(); for k in db_keys { predicate_db_conn.del::<_, ()>(&k).unwrap(); } @@ -292,7 +293,7 @@ pub fn get_chainhook_config( }; Config { http_api: PredicatesApi::On(api_config), - predicates: PredicatesConfig { payload_http_request_timeout_ms: None }, + predicates: PredicatesConfig::default(), pox_config: PoxConfig::devnet_default(), storage: StorageConfig { working_dir: working_dir.into(), @@ -343,12 +344,7 @@ pub async fn start_chainhook_service( ); let _ = hiro_system_kit::nestable_block_on(future); }) - .map_err(|e| { - format!( - "failed to start chainhook service thread, {}", - e - ) - })?; + .map_err(|e| format!("failed to start chainhook service thread, {}", e))?; // Loop to check if the server is ready let mut attempts = 0; diff --git a/components/chainhook-cli/src/service/tests/observer_tests.rs b/components/chainhook-cli/src/service/tests/observer_tests.rs index 79b51cc64..bfb512c03 100644 --- a/components/chainhook-cli/src/service/tests/observer_tests.rs +++ b/components/chainhook-cli/src/service/tests/observer_tests.rs @@ -2,7 +2,7 @@ use std::{sync::mpsc::channel, thread::sleep, time::Duration}; use chainhook_sdk::{ chainhooks::types::ChainhookStore, - observer::{start_event_observer, EventObserverConfig}, + observer::{start_event_observer, EventObserverConfig, PredicatesConfig}, types::{BitcoinNetwork, StacksNodeConfig}, utils::Context, }; @@ -190,7 +190,7 @@ async fn it_responds_200_for_unimplemented_endpoints( }); let config = EventObserverConfig { registered_chainhooks: ChainhookStore::new(), - predicate_payload_http_request_timeout_ms: None, + predicates_config: PredicatesConfig::default(), bitcoin_rpc_proxy_enabled: false, bitcoind_rpc_username: String::new(), bitcoind_rpc_password: String::new(), diff --git a/components/chainhook-sdk/src/chainhooks/bitcoin/mod.rs b/components/chainhook-sdk/src/chainhooks/bitcoin/mod.rs index e65c3add5..17959b04d 100644 --- a/components/chainhook-sdk/src/chainhooks/bitcoin/mod.rs +++ b/components/chainhook-sdk/src/chainhooks/bitcoin/mod.rs @@ -2,7 +2,7 @@ use super::types::{ append_error_context, validate_txid, ChainhookInstance, ExactMatchingRule, HookAction, MatchingRule, PoxConfig, TxinPredicate, }; -use crate::{observer::EventObserverConfig, utils::{Context, MAX_BLOCK_HEIGHTS_ENTRIES}}; +use crate::{observer::PredicatesConfig, utils::{Context, MAX_BLOCK_HEIGHTS_ENTRIES}}; use bitcoincore_rpc_json::bitcoin::{address::Payload, Address}; use chainhook_types::{ @@ -760,12 +760,12 @@ pub fn serialize_bitcoin_transactions_to_json( pub fn handle_bitcoin_hook_action<'a>( trigger: BitcoinTriggerChainhook<'a>, proofs: &HashMap<&'a TransactionIdentifier, String>, - config: &EventObserverConfig, + config: &PredicatesConfig, ) -> Result { match &trigger.chainhook.action { HookAction::HttpPost(http) => { let mut client_builder = Client::builder(); - if let Some(timeout) = config.predicate_payload_http_request_timeout_ms { + if let Some(timeout) = config.payload_http_request_timeout_ms { client_builder = client_builder.timeout(Duration::from_millis(timeout)); } let client = client_builder diff --git a/components/chainhook-sdk/src/chainhooks/stacks/mod.rs b/components/chainhook-sdk/src/chainhooks/stacks/mod.rs index 2b7723ea6..9de1e757d 100644 --- a/components/chainhook-sdk/src/chainhooks/stacks/mod.rs +++ b/components/chainhook-sdk/src/chainhooks/stacks/mod.rs @@ -1,4 +1,4 @@ -use crate::observer::EventObserverConfig; +use crate::observer::PredicatesConfig; use crate::utils::{AbstractStacksBlock, Context, MAX_BLOCK_HEIGHTS_ENTRIES}; use super::types::{ @@ -1327,13 +1327,13 @@ pub fn serialize_stacks_payload_to_json<'a>( pub fn handle_stacks_hook_action<'a>( trigger: StacksTriggerChainhook<'a>, proofs: &HashMap<&'a TransactionIdentifier, String>, - config: &EventObserverConfig, + config: &PredicatesConfig, ctx: &Context, ) -> Result { match &trigger.chainhook.action { HookAction::HttpPost(http) => { let mut client_builder = Client::builder(); - if let Some(timeout) = config.predicate_payload_http_request_timeout_ms { + if let Some(timeout) = config.payload_http_request_timeout_ms { client_builder = client_builder.timeout(Duration::from_millis(timeout)); } let client = client_builder diff --git a/components/chainhook-sdk/src/chainhooks/tests/mod.rs b/components/chainhook-sdk/src/chainhooks/tests/mod.rs index 00fa5d6bf..cd3399ab8 100644 --- a/components/chainhook-sdk/src/chainhooks/tests/mod.rs +++ b/components/chainhook-sdk/src/chainhooks/tests/mod.rs @@ -12,7 +12,10 @@ use super::{ }, types::{ExactMatchingRule, FileHook}, }; -use crate::{chainhooks::stacks::serialize_stacks_payload_to_json, utils::Context}; +use crate::{ + chainhooks::stacks::serialize_stacks_payload_to_json, observer::PredicatesConfig, + utils::Context, +}; use crate::{ chainhooks::{ tests::fixtures::{get_expected_occurrence, get_test_event_payload_by_type}, @@ -735,7 +738,8 @@ fn test_stacks_hook_action_noop() { logger: None, tracer: false, }; - let occurrence = handle_stacks_hook_action(trigger, &proofs, &ctx).unwrap(); + let occurrence = + handle_stacks_hook_action(trigger, &proofs, &PredicatesConfig::default(), &ctx).unwrap(); if let StacksChainhookOccurrence::Data(data) = occurrence { assert_eq!(data.apply.len(), 1); assert_eq!( @@ -812,7 +816,8 @@ fn test_stacks_hook_action_file_append() { logger: None, tracer: false, }; - let occurrence = handle_stacks_hook_action(trigger, &proofs, &ctx).unwrap(); + let occurrence = + handle_stacks_hook_action(trigger, &proofs, &PredicatesConfig::default(), &ctx).unwrap(); if let StacksChainhookOccurrence::File(path, bytes) = occurrence { assert_eq!(path, "./".to_string()); let json: JsonValue = serde_json::from_slice(&bytes).unwrap(); diff --git a/components/chainhook-sdk/src/observer/mod.rs b/components/chainhook-sdk/src/observer/mod.rs index 65ed619aa..25d705d3d 100644 --- a/components/chainhook-sdk/src/observer/mod.rs +++ b/components/chainhook-sdk/src/observer/mod.rs @@ -66,10 +66,29 @@ pub enum DataHandlerEvent { Terminate, } +#[derive(Clone, Debug, PartialEq)] +pub struct PredicatesConfig { + pub payload_http_request_timeout_ms: Option, +} + +impl PredicatesConfig { + pub fn new() -> Self { + PredicatesConfig { + payload_http_request_timeout_ms: None, + } + } +} + +impl Default for PredicatesConfig { + fn default() -> Self { + Self::new() + } +} + #[derive(Debug, Clone)] pub struct EventObserverConfig { pub registered_chainhooks: ChainhookStore, - pub predicate_payload_http_request_timeout_ms: Option, + pub predicates_config: PredicatesConfig, pub bitcoin_rpc_proxy_enabled: bool, pub bitcoind_rpc_username: String, pub bitcoind_rpc_password: String, @@ -291,7 +310,9 @@ impl BitcoinEventObserverConfigBuilder { }; Ok(EventObserverConfig { registered_chainhooks: ChainhookStore::new(), - predicate_payload_http_request_timeout_ms: None, + predicates_config: PredicatesConfig { + payload_http_request_timeout_ms: None, + }, bitcoin_rpc_proxy_enabled: false, bitcoind_rpc_username: self .bitcoind_rpc_username @@ -322,7 +343,9 @@ impl EventObserverConfig { pub fn default() -> Self { EventObserverConfig { registered_chainhooks: ChainhookStore::new(), - predicate_payload_http_request_timeout_ms: None, + predicates_config: PredicatesConfig { + payload_http_request_timeout_ms: None, + }, bitcoin_rpc_proxy_enabled: false, bitcoind_rpc_username: "devnet".into(), bitcoind_rpc_password: "devnet".into(), @@ -366,7 +389,6 @@ impl EventObserverConfig { } pub fn get_bitcoin_config(&self) -> BitcoinConfig { - BitcoinConfig { username: self.bitcoind_rpc_username.clone(), password: self.bitcoind_rpc_password.clone(), @@ -406,7 +428,9 @@ impl EventObserverConfig { let config = EventObserverConfig { bitcoin_rpc_proxy_enabled: false, registered_chainhooks: ChainhookStore::new(), - predicate_payload_http_request_timeout_ms: None, + predicates_config: PredicatesConfig { + payload_http_request_timeout_ms: None, + }, bitcoind_rpc_username: overrides .and_then(|c| c.bitcoind_rpc_username.clone()) .unwrap_or_else(|| "devnet".to_string()), @@ -1053,8 +1077,7 @@ pub fn get_bitcoin_proof( Ok(proof) => Ok(format!("0x{}", hex::encode(&proof))), Err(e) => Err(format!( "failed collecting proof for transaction {}: {}", - transaction_identifier.hash, - e + transaction_identifier.hash, e )), } } @@ -1365,7 +1388,8 @@ pub async fn start_observer_commands_handler( if let Some(highest_tip_block) = blocks_to_apply .iter() - .max_by_key(|b| b.block_identifier.index) { + .max_by_key(|b| b.block_identifier.index) + { prometheus_monitoring.btc_metrics_set_reorg( highest_tip_block.timestamp.into(), blocks_to_apply.len() as u64, @@ -1483,7 +1507,11 @@ pub async fn start_observer_commands_handler( } for chainhook_to_trigger in chainhooks_to_trigger.into_iter() { let predicate_uuid = &chainhook_to_trigger.chainhook.uuid; - match handle_bitcoin_hook_action(chainhook_to_trigger, &proofs, &config) { + match handle_bitcoin_hook_action( + chainhook_to_trigger, + &proofs, + &config.predicates_config, + ) { Err(e) => { // todo: we may want to set predicates that reach this branch as interrupted, // but for now we will error to see if this problem occurs. @@ -1672,7 +1700,12 @@ pub async fn start_observer_commands_handler( let proofs = HashMap::new(); for chainhook_to_trigger in chainhooks_to_trigger.into_iter() { let predicate_uuid = &chainhook_to_trigger.chainhook.uuid; - match handle_stacks_hook_action(chainhook_to_trigger, &proofs, &config, &ctx) { + match handle_stacks_hook_action( + chainhook_to_trigger, + &proofs, + &config.predicates_config, + &ctx, + ) { Err(e) => { ctx.try_log(|logger| { // todo: we may want to set predicates that reach this branch as interrupted, diff --git a/components/chainhook-sdk/src/observer/tests/mod.rs b/components/chainhook-sdk/src/observer/tests/mod.rs index b14538331..41bcd25d9 100644 --- a/components/chainhook-sdk/src/observer/tests/mod.rs +++ b/components/chainhook-sdk/src/observer/tests/mod.rs @@ -35,12 +35,13 @@ use hiro_system_kit; use std::collections::BTreeMap; use std::sync::mpsc::{channel, Sender}; +use super::PredicatesConfig; use super::{ObserverEvent, DEFAULT_INGESTION_PORT}; fn generate_test_config() -> (EventObserverConfig, ChainhookStore) { let config: EventObserverConfig = EventObserverConfig { registered_chainhooks: ChainhookStore::new(), - predicate_payload_http_request_timeout_ms: None, + predicates_config: PredicatesConfig::default(), bitcoin_rpc_proxy_enabled: false, bitcoind_rpc_username: "user".into(), bitcoind_rpc_password: "user".into(), From 93df40525f555dded1b510ebfb0d18c8c6415cbb Mon Sep 17 00:00:00 2001 From: Rafael Cardenas Date: Thu, 10 Oct 2024 10:34:15 -0600 Subject: [PATCH 4/4] fix: config style --- components/chainhook-cli/src/scan/bitcoin.rs | 2 +- components/chainhook-cli/src/scan/stacks.rs | 10 +++------- .../chainhook-sdk/src/chainhooks/bitcoin/mod.rs | 6 +++--- .../chainhook-sdk/src/chainhooks/stacks/mod.rs | 6 +++--- .../chainhook-sdk/src/chainhooks/tests/mod.rs | 7 ++++--- components/chainhook-sdk/src/observer/mod.rs | 13 ++----------- 6 files changed, 16 insertions(+), 28 deletions(-) diff --git a/components/chainhook-cli/src/scan/bitcoin.rs b/components/chainhook-cli/src/scan/bitcoin.rs index fef4e507e..3427ef60b 100644 --- a/components/chainhook-cli/src/scan/bitcoin.rs +++ b/components/chainhook-cli/src/scan/bitcoin.rs @@ -290,7 +290,7 @@ pub async fn execute_predicates_action<'a>( gather_proofs(&trigger, &mut proofs, config, ctx); } let predicate_uuid = &trigger.chainhook.uuid; - match handle_bitcoin_hook_action(trigger, &proofs, &config.predicates_config) { + match handle_bitcoin_hook_action(trigger, &proofs, &config) { Err(e) => { warn!( ctx.expect_logger(), diff --git a/components/chainhook-cli/src/scan/stacks.rs b/components/chainhook-cli/src/scan/stacks.rs index 279775b74..1e3b100e5 100644 --- a/components/chainhook-cli/src/scan/stacks.rs +++ b/components/chainhook-cli/src/scan/stacks.rs @@ -343,7 +343,7 @@ pub async fn scan_stacks_chainstate_via_rocksdb_using_predicate( let res = match handle_stacks_hook_action( trigger, &proofs, - &config.get_event_observer_config().predicates_config, + &config.get_event_observer_config(), ctx, ) { Err(e) => { @@ -534,12 +534,8 @@ pub async fn scan_stacks_chainstate_via_csv_using_predicate( apply: hits_per_blocks, rollback: vec![], }; - match handle_stacks_hook_action( - trigger, - &proofs, - &config.get_event_observer_config().predicates_config, - ctx, - ) { + match handle_stacks_hook_action(trigger, &proofs, &config.get_event_observer_config(), ctx) + { Err(e) => { error!(ctx.expect_logger(), "unable to handle action {}", e); } diff --git a/components/chainhook-sdk/src/chainhooks/bitcoin/mod.rs b/components/chainhook-sdk/src/chainhooks/bitcoin/mod.rs index 17959b04d..e239521c2 100644 --- a/components/chainhook-sdk/src/chainhooks/bitcoin/mod.rs +++ b/components/chainhook-sdk/src/chainhooks/bitcoin/mod.rs @@ -2,7 +2,7 @@ use super::types::{ append_error_context, validate_txid, ChainhookInstance, ExactMatchingRule, HookAction, MatchingRule, PoxConfig, TxinPredicate, }; -use crate::{observer::PredicatesConfig, utils::{Context, MAX_BLOCK_HEIGHTS_ENTRIES}}; +use crate::{observer::EventObserverConfig, utils::{Context, MAX_BLOCK_HEIGHTS_ENTRIES}}; use bitcoincore_rpc_json::bitcoin::{address::Payload, Address}; use chainhook_types::{ @@ -760,12 +760,12 @@ pub fn serialize_bitcoin_transactions_to_json( pub fn handle_bitcoin_hook_action<'a>( trigger: BitcoinTriggerChainhook<'a>, proofs: &HashMap<&'a TransactionIdentifier, String>, - config: &PredicatesConfig, + config: &EventObserverConfig, ) -> Result { match &trigger.chainhook.action { HookAction::HttpPost(http) => { let mut client_builder = Client::builder(); - if let Some(timeout) = config.payload_http_request_timeout_ms { + if let Some(timeout) = config.predicates_config.payload_http_request_timeout_ms { client_builder = client_builder.timeout(Duration::from_millis(timeout)); } let client = client_builder diff --git a/components/chainhook-sdk/src/chainhooks/stacks/mod.rs b/components/chainhook-sdk/src/chainhooks/stacks/mod.rs index 9de1e757d..1b8e5bfa9 100644 --- a/components/chainhook-sdk/src/chainhooks/stacks/mod.rs +++ b/components/chainhook-sdk/src/chainhooks/stacks/mod.rs @@ -1,4 +1,4 @@ -use crate::observer::PredicatesConfig; +use crate::observer::EventObserverConfig; use crate::utils::{AbstractStacksBlock, Context, MAX_BLOCK_HEIGHTS_ENTRIES}; use super::types::{ @@ -1327,13 +1327,13 @@ pub fn serialize_stacks_payload_to_json<'a>( pub fn handle_stacks_hook_action<'a>( trigger: StacksTriggerChainhook<'a>, proofs: &HashMap<&'a TransactionIdentifier, String>, - config: &PredicatesConfig, + config: &EventObserverConfig, ctx: &Context, ) -> Result { match &trigger.chainhook.action { HookAction::HttpPost(http) => { let mut client_builder = Client::builder(); - if let Some(timeout) = config.payload_http_request_timeout_ms { + if let Some(timeout) = config.predicates_config.payload_http_request_timeout_ms { client_builder = client_builder.timeout(Duration::from_millis(timeout)); } let client = client_builder diff --git a/components/chainhook-sdk/src/chainhooks/tests/mod.rs b/components/chainhook-sdk/src/chainhooks/tests/mod.rs index cd3399ab8..9c5ee7625 100644 --- a/components/chainhook-sdk/src/chainhooks/tests/mod.rs +++ b/components/chainhook-sdk/src/chainhooks/tests/mod.rs @@ -13,7 +13,8 @@ use super::{ types::{ExactMatchingRule, FileHook}, }; use crate::{ - chainhooks::stacks::serialize_stacks_payload_to_json, observer::PredicatesConfig, + chainhooks::stacks::serialize_stacks_payload_to_json, + observer::EventObserverConfig, utils::Context, }; use crate::{ @@ -739,7 +740,7 @@ fn test_stacks_hook_action_noop() { tracer: false, }; let occurrence = - handle_stacks_hook_action(trigger, &proofs, &PredicatesConfig::default(), &ctx).unwrap(); + handle_stacks_hook_action(trigger, &proofs, &EventObserverConfig::default(), &ctx).unwrap(); if let StacksChainhookOccurrence::Data(data) = occurrence { assert_eq!(data.apply.len(), 1); assert_eq!( @@ -817,7 +818,7 @@ fn test_stacks_hook_action_file_append() { tracer: false, }; let occurrence = - handle_stacks_hook_action(trigger, &proofs, &PredicatesConfig::default(), &ctx).unwrap(); + handle_stacks_hook_action(trigger, &proofs, &EventObserverConfig::default(), &ctx).unwrap(); if let StacksChainhookOccurrence::File(path, bytes) = occurrence { assert_eq!(path, "./".to_string()); let json: JsonValue = serde_json::from_slice(&bytes).unwrap(); diff --git a/components/chainhook-sdk/src/observer/mod.rs b/components/chainhook-sdk/src/observer/mod.rs index 25d705d3d..776e2e781 100644 --- a/components/chainhook-sdk/src/observer/mod.rs +++ b/components/chainhook-sdk/src/observer/mod.rs @@ -1507,11 +1507,7 @@ pub async fn start_observer_commands_handler( } for chainhook_to_trigger in chainhooks_to_trigger.into_iter() { let predicate_uuid = &chainhook_to_trigger.chainhook.uuid; - match handle_bitcoin_hook_action( - chainhook_to_trigger, - &proofs, - &config.predicates_config, - ) { + match handle_bitcoin_hook_action(chainhook_to_trigger, &proofs, &config) { Err(e) => { // todo: we may want to set predicates that reach this branch as interrupted, // but for now we will error to see if this problem occurs. @@ -1700,12 +1696,7 @@ pub async fn start_observer_commands_handler( let proofs = HashMap::new(); for chainhook_to_trigger in chainhooks_to_trigger.into_iter() { let predicate_uuid = &chainhook_to_trigger.chainhook.uuid; - match handle_stacks_hook_action( - chainhook_to_trigger, - &proofs, - &config.predicates_config, - &ctx, - ) { + match handle_stacks_hook_action(chainhook_to_trigger, &proofs, &config, &ctx) { Err(e) => { ctx.try_log(|logger| { // todo: we may want to set predicates that reach this branch as interrupted,