From 2dd40064199ccc64230e45b06d6acf88985f6ea3 Mon Sep 17 00:00:00 2001 From: Rafael Cardenas Date: Fri, 18 Oct 2024 09:50:16 -0600 Subject: [PATCH 01/15] chore: upgrade codec --- Cargo.lock | 5 ++--- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 17b1c93ff..5f01348f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3538,12 +3538,11 @@ dependencies = [ [[package]] name = "stacks-codec" -version = "2.7.0" -source = "git+https://github.com/hirosystems/clarinet.git?rev=3a2f9136abd85b265e538fbe51c808e9c09a06cb#3a2f9136abd85b265e538fbe51c808e9c09a06cb" +version = "2.9.0" +source = "git+https://github.com/hirosystems/clarinet.git?rev=b0683675115562d719ed4b5245f620e0990030a0#b0683675115562d719ed4b5245f620e0990030a0" dependencies = [ "clarity", "serde", - "wsts", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 2111714ed..e7694ce18 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,4 +8,4 @@ default-members = ["components/chainhook-cli", "components/chainhook-sdk"] resolver = "2" [patch.crates-io] -stacks-codec = { git = "https://github.com/hirosystems/clarinet.git", rev = "3a2f9136abd85b265e538fbe51c808e9c09a06cb" } +stacks-codec = { git = "https://github.com/hirosystems/clarinet.git", rev = "b0683675115562d719ed4b5245f620e0990030a0" } From 5b8232a00564fdaf3fae96b2c4d10d334aa63843 Mon Sep 17 00:00:00 2001 From: Rafael Cardenas Date: Fri, 18 Oct 2024 09:50:31 -0600 Subject: [PATCH 02/15] feat: signer predicates --- components/chainhook-cli/src/scan/stacks.rs | 3 + components/chainhook-cli/src/service/mod.rs | 2 +- .../src/chainhooks/stacks/mod.rs | 117 +++++++++++++++--- .../chainhook-sdk/src/chainhooks/tests/mod.rs | 2 + components/chainhook-sdk/src/indexer/mod.rs | 13 +- .../src/indexer/stacks/blocks_pool.rs | 9 ++ .../chainhook-sdk/src/indexer/stacks/mod.rs | 9 +- .../chainhook-sdk/src/indexer/stacks/tests.rs | 12 +- components/chainhook-sdk/src/utils/mod.rs | 15 ++- components/chainhook-types-rs/src/rosetta.rs | 2 + components/chainhook-types-rs/src/signers.rs | 5 +- .../typescript/src/schemas/stacks/if_this.ts | 22 ++++ 12 files changed, 179 insertions(+), 32 deletions(-) diff --git a/components/chainhook-cli/src/scan/stacks.rs b/components/chainhook-cli/src/scan/stacks.rs index 1e3b100e5..eda01a0a2 100644 --- a/components/chainhook-cli/src/scan/stacks.rs +++ b/components/chainhook-cli/src/scan/stacks.rs @@ -339,6 +339,7 @@ pub async fn scan_stacks_chainstate_via_rocksdb_using_predicate( chainhook: predicate_spec, apply: hits_per_blocks, rollback: vec![], + events: vec![] }; let res = match handle_stacks_hook_action( trigger, @@ -533,6 +534,8 @@ pub async fn scan_stacks_chainstate_via_csv_using_predicate( chainhook: predicate_spec, apply: hits_per_blocks, rollback: vec![], + // TODO(rafaelcr): Consider StackerDB chunks that come from TSVs. + events: vec![] }; match handle_stacks_hook_action(trigger, &proofs, &config.get_event_observer_config(), ctx) { diff --git a/components/chainhook-cli/src/service/mod.rs b/components/chainhook-cli/src/service/mod.rs index a6249a4c5..a994b3365 100644 --- a/components/chainhook-cli/src/service/mod.rs +++ b/components/chainhook-cli/src/service/mod.rs @@ -619,7 +619,7 @@ impl Service { } StacksChainEvent::ChainUpdatedWithMicroblocks(_) | StacksChainEvent::ChainUpdatedWithMicroblocksReorg(_) => {}, - StacksChainEvent::ChainUpdatedWithStackerDbChunks(data) => { + StacksChainEvent::ChainUpdatedWithStackerDbChunks(_) => { // TODO(rafaelcr): Send via HTTP payload. }, }; diff --git a/components/chainhook-sdk/src/chainhooks/stacks/mod.rs b/components/chainhook-sdk/src/chainhooks/stacks/mod.rs index 66f2205c4..0433851f1 100644 --- a/components/chainhook-sdk/src/chainhooks/stacks/mod.rs +++ b/components/chainhook-sdk/src/chainhooks/stacks/mod.rs @@ -1,13 +1,15 @@ use crate::observer::EventObserverConfig; -use crate::utils::{AbstractStacksBlock, Context, MAX_BLOCK_HEIGHTS_ENTRIES}; +use crate::utils::{ + AbstractStacksBlock, AbstractStacksNonConsensusEvent, Context, MAX_BLOCK_HEIGHTS_ENTRIES, +}; +use super::types::validate_txid; use super::types::{ append_error_context, BlockIdentifierIndexRule, ChainhookInstance, ExactMatchingRule, HookAction, }; -use super::types::validate_txid; use chainhook_types::{ - BlockIdentifier, StacksChainEvent, StacksNetwork, StacksTransactionData, + BlockIdentifier, StacksChainEvent, StacksNetwork, StacksStackerDbChunk, StacksTransactionData, StacksTransactionEvent, StacksTransactionEventPayload, StacksTransactionKind, TransactionIdentifier, }; @@ -259,6 +261,8 @@ pub enum StacksPredicate { NftEvent(StacksNftEventBasedPredicate), StxEvent(StacksStxEventBasedPredicate), Txid(ExactMatchingRule), + #[cfg(feature = "stacks-signers")] + SignerMessage(StacksSignerMessagePredicate), } impl StacksPredicate { @@ -307,11 +311,28 @@ impl StacksPredicate { )); } } + #[cfg(feature = "stacks-signers")] + StacksPredicate::SignerMessage(StacksSignerMessagePredicate::FromSignerPubKey(_)) => { + // TODO(rafaelcr): Validate pubkey format + } + #[cfg(feature = "stacks-signers")] + StacksPredicate::SignerMessage(_) => {} } Ok(()) } } +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum StacksSignerMessagePredicate { + AfterTimestamp(u64), + FromSignerPubKey(String), +} + +impl StacksSignerMessagePredicate { + // TODO(rafaelcr): Write validators +} + #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, JsonSchema)] #[serde(rename_all = "snake_case")] pub struct StacksContractCallBasedPredicate { @@ -457,6 +478,7 @@ pub struct StacksTriggerChainhook<'a> { pub chainhook: &'a StacksChainhookInstance, pub apply: Vec<(Vec<&'a StacksTransactionData>, &'a dyn AbstractStacksBlock)>, pub rollback: Vec<(Vec<&'a StacksTransactionData>, &'a dyn AbstractStacksBlock)>, + pub events: Vec<&'a dyn AbstractStacksNonConsensusEvent>, } #[derive(Clone, Debug)] @@ -484,17 +506,13 @@ pub struct StacksChainhookOccurrencePayload { } impl StacksChainhookOccurrencePayload { - pub fn from_trigger( - trigger: StacksTriggerChainhook<'_>, - ) -> StacksChainhookOccurrencePayload { + pub fn from_trigger(trigger: StacksTriggerChainhook<'_>) -> StacksChainhookOccurrencePayload { StacksChainhookOccurrencePayload { apply: trigger .apply .into_iter() .map(|(transactions, block)| { - let transactions = transactions - .into_iter().cloned() - .collect::>(); + let transactions = transactions.into_iter().cloned().collect::>(); StacksApplyTransactionPayload { block_identifier: block.get_identifier().clone(), transactions, @@ -505,9 +523,7 @@ impl StacksChainhookOccurrencePayload { .rollback .into_iter() .map(|(transactions, block)| { - let transactions = transactions - .into_iter().cloned() - .collect::>(); + let transactions = transactions.into_iter().cloned().collect::>(); StacksRollbackTransactionPayload { block_identifier: block.get_identifier().clone(), transactions, @@ -593,6 +609,7 @@ pub fn evaluate_stacks_chainhooks_on_chain_event<'a>( chainhook, apply, rollback, + events: vec![], }) } } @@ -621,6 +638,7 @@ pub fn evaluate_stacks_chainhooks_on_chain_event<'a>( chainhook, apply, rollback, + events: vec![], }) } } @@ -657,6 +675,7 @@ pub fn evaluate_stacks_chainhooks_on_chain_event<'a>( chainhook, apply, rollback, + events: vec![], }) } } @@ -718,13 +737,35 @@ pub fn evaluate_stacks_chainhooks_on_chain_event<'a>( chainhook, apply, rollback, + events: vec![], }) } } - }, + } StacksChainEvent::ChainUpdatedWithStackerDbChunks(data) => { - // TODO: Support predicates to send this data - }, + for chainhook in active_chainhooks.iter() { + let mut events = vec![]; + + evaluated_predicates.insert(chainhook.uuid.as_str(), &data.received_at_block); + let mut chunks: Vec<&dyn AbstractStacksNonConsensusEvent> = vec![]; + for chunk in data.chunks.iter() { + chunks.push(chunk); + } + let (mut occurrences, mut expirations) = + evaluate_stacks_predicate_on_stackerdb_chunks(chunks, chainhook, ctx); + events.append(&mut occurrences); + expired_predicates.append(&mut expirations); + + if events.len() > 0 { + triggered_predicates.push(StacksTriggerChainhook { + chainhook, + apply: vec![], + rollback: vec![], + events, + }); + } + } + } } ( triggered_predicates, @@ -795,7 +836,45 @@ pub fn evaluate_stacks_predicate_on_block<'a>( | StacksPredicate::StxEvent(_) | StacksPredicate::PrintEvent(_) | StacksPredicate::Txid(_) => unreachable!(), + #[cfg(feature = "stacks-signers")] + StacksPredicate::SignerMessage(_) => unreachable!(), + } +} + +#[cfg(feature = "stacks-signers")] +pub fn evaluate_stacks_predicate_on_stackerdb_chunks<'a>( + chunks: Vec<&'a dyn AbstractStacksNonConsensusEvent>, + chainhook: &'a StacksChainhookInstance, + _ctx: &Context, +) -> ( + Vec<&'a dyn AbstractStacksNonConsensusEvent>, + BTreeMap<&'a str, &'a BlockIdentifier>, +) { + let mut occurrences = vec![]; + let expired_predicates = BTreeMap::new(); + for chunk in chunks { + match &chainhook.predicate { + StacksPredicate::SignerMessage(StacksSignerMessagePredicate::AfterTimestamp( + timestamp, + )) => { + if chunk.get_timestamp() >= *timestamp as i64 { + occurrences.push(chunk); + } + } + StacksPredicate::SignerMessage(StacksSignerMessagePredicate::FromSignerPubKey(_)) => { + todo!() + } + StacksPredicate::BlockHeight(_) + | StacksPredicate::ContractDeployment(_) + | StacksPredicate::ContractCall(_) + | StacksPredicate::FtEvent(_) + | StacksPredicate::NftEvent(_) + | StacksPredicate::StxEvent(_) + | StacksPredicate::PrintEvent(_) + | StacksPredicate::Txid(_) => unreachable!(), + }; } + (occurrences, expired_predicates) } pub fn evaluate_stacks_predicate_on_transaction<'a>( @@ -952,7 +1031,9 @@ pub fn evaluate_stacks_predicate_on_transaction<'a>( } StacksPredicate::PrintEvent(expected_event) => { for event in transaction.metadata.receipt.events.iter() { - if let StacksTransactionEventPayload::SmartContractEvent(actual) = &event.event_payload { + if let StacksTransactionEventPayload::SmartContractEvent(actual) = + &event.event_payload + { if actual.topic == "print" { match expected_event { StacksPrintEventBasedPredicate::Contains { @@ -1006,6 +1087,8 @@ pub fn evaluate_stacks_predicate_on_transaction<'a>( txid.eq(&transaction.transaction_identifier.hash) } StacksPredicate::BlockHeight(_) => unreachable!(), + #[cfg(feature = "stacks-signers")] + StacksPredicate::SignerMessage(_) => unreachable!(), } } @@ -1238,7 +1321,7 @@ pub fn serialized_decoded_clarity_value(hex_value: &str, ctx: &Context) -> serde Ok(bytes) => bytes, _ => return json!(hex_value.to_string()), }; - + match ClarityValue::consensus_deserialize(&mut Cursor::new(&value_bytes)) { Ok(value) => serialize_to_json(&value), Err(e) => { diff --git a/components/chainhook-sdk/src/chainhooks/tests/mod.rs b/components/chainhook-sdk/src/chainhooks/tests/mod.rs index 9c5ee7625..2370a8f29 100644 --- a/components/chainhook-sdk/src/chainhooks/tests/mod.rs +++ b/components/chainhook-sdk/src/chainhooks/tests/mod.rs @@ -732,6 +732,7 @@ fn test_stacks_hook_action_noop() { chainhook: &chainhook, apply: vec![(apply_transactions, apply_blocks)], rollback: vec![(rollback_transactions, rollback_blocks)], + events: vec![] }; let proofs = HashMap::new(); @@ -810,6 +811,7 @@ fn test_stacks_hook_action_file_append() { chainhook: &chainhook, apply, rollback: vec![(rollback_transactions, rollback_block)], + events: vec![] }; let proofs = HashMap::new(); diff --git a/components/chainhook-sdk/src/indexer/mod.rs b/components/chainhook-sdk/src/indexer/mod.rs index af484d466..9de41545b 100644 --- a/components/chainhook-sdk/src/indexer/mod.rs +++ b/components/chainhook-sdk/src/indexer/mod.rs @@ -175,17 +175,22 @@ impl Indexer { ctx: &Context, ) -> Result, String> { use chainhook_types::StacksChainUpdatedWithStackerDbChunksData; - + let Some(chain_tip) = self.stacks_blocks_pool.get_canonical_fork_chain_tip() else { + return Err("StackerDB chunk received with no canonical chain tip".to_string()); + }; let chunks = stacks::standardize_stacks_marshalled_stackerdb_chunks( - &self.config, marshalled_stackerdb_chunks, receipt_time, - &mut self.stacks_context, + chain_tip, ctx, )?; if chunks.len() > 0 { Ok(Some(StacksChainEvent::ChainUpdatedWithStackerDbChunks( - StacksChainUpdatedWithStackerDbChunksData { chunks }, + StacksChainUpdatedWithStackerDbChunksData { + chunks, + received_at: receipt_time, + received_at_block: chain_tip.clone(), + }, ))) } else { Ok(None) diff --git a/components/chainhook-sdk/src/indexer/stacks/blocks_pool.rs b/components/chainhook-sdk/src/indexer/stacks/blocks_pool.rs index 5897144c2..80cf44b94 100644 --- a/components/chainhook-sdk/src/indexer/stacks/blocks_pool.rs +++ b/components/chainhook-sdk/src/indexer/stacks/blocks_pool.rs @@ -49,6 +49,15 @@ impl StacksBlockPool { } } + pub fn get_canonical_fork_chain_tip(&self) -> Option<&BlockIdentifier> { + match self.forks.get(&self.canonical_fork_id) { + Some(fork) => { + Some(fork.get_tip()) + }, + None => None, + } + } + pub fn seed_block_pool(&mut self, blocks: Vec, ctx: &Context) { ctx.try_log(|logger| { slog::info!(logger, "Seeding block pool with {} blocks", blocks.len()) diff --git a/components/chainhook-sdk/src/indexer/stacks/mod.rs b/components/chainhook-sdk/src/indexer/stacks/mod.rs index 142e18bae..5d7263d5b 100644 --- a/components/chainhook-sdk/src/indexer/stacks/mod.rs +++ b/components/chainhook-sdk/src/indexer/stacks/mod.rs @@ -593,22 +593,22 @@ pub fn standardize_stacks_microblock_trail( #[cfg(feature = "stacks-signers")] pub fn standardize_stacks_marshalled_stackerdb_chunks( - _indexer_config: &IndexerConfig, marshalled_stackerdb_chunks: JsonValue, receipt_time: u64, - _chain_ctx: &mut StacksChainContext, + chain_tip: &BlockIdentifier, _ctx: &Context, ) -> Result, String> { let mut stackerdb_chunks: NewStackerDbChunks = serde_json::from_value(marshalled_stackerdb_chunks) .map_err(|e| format!("unable to parse stackerdb chunks {e}"))?; - standardize_stacks_stackerdb_chunks(&mut stackerdb_chunks, receipt_time) + standardize_stacks_stackerdb_chunks(&mut stackerdb_chunks, receipt_time, chain_tip) } #[cfg(feature = "stacks-signers")] pub fn standardize_stacks_stackerdb_chunks( stackerdb_chunks: &NewStackerDbChunks, receipt_time: u64, + chain_tip: &BlockIdentifier, ) -> Result, String> { use stacks_codec::codec::BlockResponse; use stacks_codec::codec::RejectCode; @@ -705,7 +705,8 @@ pub fn standardize_stacks_stackerdb_chunks( sig: slot.sig.clone(), pubkey: get_signer_pubkey_from_stackerdb_chunk_slot(slot, &data_bytes)?, message, - receipt_time, + received_at: receipt_time, + received_at_block: chain_tip.clone() }); } diff --git a/components/chainhook-sdk/src/indexer/stacks/tests.rs b/components/chainhook-sdk/src/indexer/stacks/tests.rs index e45be1648..b129121bc 100644 --- a/components/chainhook-sdk/src/indexer/stacks/tests.rs +++ b/components/chainhook-sdk/src/indexer/stacks/tests.rs @@ -402,7 +402,7 @@ fn into_chainhook_event_rejects_invalid_missing_event() { #[test] #[cfg(feature = "stacks-signers")] fn stackerdb_chunks_covert_into_signer_messages() { - use chainhook_types::{BlockResponseData, StacksSignerMessage}; + use chainhook_types::{BlockIdentifier, BlockResponseData, StacksSignerMessage}; use crate::indexer::tests::helpers::stacks_events::create_new_stackerdb_chunk; @@ -413,7 +413,15 @@ fn stackerdb_chunks_covert_into_signer_messages() { "01fc3c06f6e0ae5b13c9bb53763661817e55c8e7f1ecab8b4d4b65b283d2dd39f0099e3ea1e25e765f4f0e1dfb0a432309a16a2ec10940e1a14cb9e9b1cbf27edc".to_string(), "010074aff146904763a787aa14c614d0dd1fc63b537bdb2fd351cdf881f6db75f986005eb55250597b25acbf99d3dd3c2fa8189046e1b5d21309a44cbaf2b327c09b0159a01ed3f0094bfa9e5f72f5d894e12ce252081eab5396eb8bba137bddfc365b".to_string() ); - let parsed_chunk = standardize_stacks_stackerdb_chunks(&new_chunks, 1729013425).unwrap(); + let parsed_chunk = standardize_stacks_stackerdb_chunks( + &new_chunks, + 1729013425, + &BlockIdentifier { + index: 170355, + hash: "0x519df2ad0d86a62dc078865486a1f3dbb2f1a8934da81b679561738820964fe0".to_string(), + }, + ) + .unwrap(); assert_eq!(parsed_chunk.len(), 1); let message = &parsed_chunk[0]; diff --git a/components/chainhook-sdk/src/utils/mod.rs b/components/chainhook-sdk/src/utils/mod.rs index 4de1df404..49cfb4ff5 100644 --- a/components/chainhook-sdk/src/utils/mod.rs +++ b/components/chainhook-sdk/src/utils/mod.rs @@ -6,8 +6,7 @@ use std::{ }; use chainhook_types::{ - BitcoinBlockData, BlockHeader, BlockIdentifier, StacksBlockData, StacksMicroblockData, - StacksTransactionData, + BitcoinBlockData, BlockHeader, BlockIdentifier, StacksBlockData, StacksMicroblockData, StacksStackerDbChunk, StacksTransactionData }; use hiro_system_kit::slog::{self, Logger}; use reqwest::RequestBuilder; @@ -93,6 +92,18 @@ impl AbstractStacksBlock for StacksMicroblockData { } } +/// Trait for Stacks events that are not part of the blockchain consensus but are otherwise broadcasted by Stacks nodes to event +/// listeners. +pub trait AbstractStacksNonConsensusEvent { + fn get_timestamp(&self) -> i64; +} + +impl AbstractStacksNonConsensusEvent for StacksStackerDbChunk { + fn get_timestamp(&self) -> i64 { + self.received_at as i64 + } +} + pub trait AbstractBlock { fn get_identifier(&self) -> &BlockIdentifier; fn get_parent_identifier(&self) -> &BlockIdentifier; diff --git a/components/chainhook-types-rs/src/rosetta.rs b/components/chainhook-types-rs/src/rosetta.rs index 804f828aa..0f778b3cb 100644 --- a/components/chainhook-types-rs/src/rosetta.rs +++ b/components/chainhook-types-rs/src/rosetta.rs @@ -670,6 +670,8 @@ pub struct BitcoinChainUpdatedWithReorgData { #[derive(Debug, Clone, PartialEq, Serialize)] pub struct StacksChainUpdatedWithStackerDbChunksData { pub chunks: Vec, + pub received_at: u64, + pub received_at_block: BlockIdentifier, } #[allow(dead_code)] diff --git a/components/chainhook-types-rs/src/signers.rs b/components/chainhook-types-rs/src/signers.rs index c1fd4e873..dd5cb6871 100644 --- a/components/chainhook-types-rs/src/signers.rs +++ b/components/chainhook-types-rs/src/signers.rs @@ -1,4 +1,4 @@ -use crate::StacksTransactionData; +use crate::{BlockIdentifier, StacksTransactionData}; #[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] pub struct NakamotoBlockHeaderData { @@ -92,5 +92,6 @@ pub struct StacksStackerDbChunk { pub sig: String, pub pubkey: String, pub message: StacksSignerMessage, - pub receipt_time: u64, + pub received_at: u64, + pub received_at_block: BlockIdentifier, } diff --git a/components/client/typescript/src/schemas/stacks/if_this.ts b/components/client/typescript/src/schemas/stacks/if_this.ts index 9f3f709eb..d23689bdb 100644 --- a/components/client/typescript/src/schemas/stacks/if_this.ts +++ b/components/client/typescript/src/schemas/stacks/if_this.ts @@ -74,6 +74,27 @@ export type StacksIfThisContractDeploymentTrait = Static< typeof StacksIfThisContractDeploymentTraitSchema >; +export const StacksIfThisSignerMessageAfterTimestampSchema = Type.Object({ + scope: Type.Literal('signer_message'), + after_timestamp: Type.Integer(), +}); +export type StacksIfThisSignerMessageAfterTimestamp = Static< + typeof StacksIfThisSignerMessageAfterTimestampSchema +>; + +export const StacksIfThisSignerMessageSignerPubKeySchema = Type.Object({ + scope: Type.Literal('signer_message'), + signer_pubkey: Type.String(), +}); +export type StacksIfThisSignerMessageSignerPubKey = Static< + typeof StacksIfThisSignerMessageSignerPubKeySchema +>; + +export const StacksIfThisSignerMessageSchema = Type.Union([ + StacksIfThisSignerMessageAfterTimestampSchema, +]); +export type StacksIfThisSignerMessage = Static; + export const StacksIfThisOptionsSchema = Type.Object({ start_block: Type.Optional(Type.Integer()), end_block: Type.Optional(Type.Integer()), @@ -93,6 +114,7 @@ export const StacksIfThisSchema = Type.Union([ StacksIfThisContractCallSchema, StacksIfThisContractDeploymentSchema, StacksIfThisContractDeploymentTraitSchema, + StacksIfThisSignerMessageSchema, ]); export type StacksIfThis = Static; From 5a6a79197e27ed1574fdb7963231e1f0999a0028 Mon Sep 17 00:00:00 2001 From: Rafael Cardenas Date: Fri, 18 Oct 2024 12:01:08 -0600 Subject: [PATCH 03/15] fix: payload start --- components/chainhook-sdk/src/chainhooks/stacks/mod.rs | 7 +++++++ components/chainhook-sdk/src/utils/mod.rs | 5 +++++ 2 files changed, 12 insertions(+) diff --git a/components/chainhook-sdk/src/chainhooks/stacks/mod.rs b/components/chainhook-sdk/src/chainhooks/stacks/mod.rs index 0433851f1..5df74c567 100644 --- a/components/chainhook-sdk/src/chainhooks/stacks/mod.rs +++ b/components/chainhook-sdk/src/chainhooks/stacks/mod.rs @@ -493,6 +493,12 @@ pub struct StacksRollbackTransactionPayload { pub transactions: Vec, } +#[derive(Clone, Debug)] +pub struct StacksNonConsensusEventPayload { + pub block_identifier: BlockIdentifier, + pub transactions: Vec, +} + #[derive(Clone, Debug)] pub struct StacksChainhookPayload { pub uuid: String, @@ -502,6 +508,7 @@ pub struct StacksChainhookPayload { pub struct StacksChainhookOccurrencePayload { pub apply: Vec, pub rollback: Vec, + pub events: Vec, pub chainhook: StacksChainhookPayload, } diff --git a/components/chainhook-sdk/src/utils/mod.rs b/components/chainhook-sdk/src/utils/mod.rs index 49cfb4ff5..f2de01d3e 100644 --- a/components/chainhook-sdk/src/utils/mod.rs +++ b/components/chainhook-sdk/src/utils/mod.rs @@ -96,12 +96,17 @@ impl AbstractStacksBlock for StacksMicroblockData { /// listeners. pub trait AbstractStacksNonConsensusEvent { fn get_timestamp(&self) -> i64; + fn get_received_at_block(&self) -> BlockIdentifier; } impl AbstractStacksNonConsensusEvent for StacksStackerDbChunk { fn get_timestamp(&self) -> i64 { self.received_at as i64 } + + fn get_received_at_block(&self) -> BlockIdentifier { + self.received_at_block.clone() + } } pub trait AbstractBlock { From f0569406a55400e4de9cb752555ecd523fa957a1 Mon Sep 17 00:00:00 2001 From: Rafael Cardenas Date: Tue, 22 Oct 2024 11:01:24 -0600 Subject: [PATCH 04/15] fix: data bubble --- components/chainhook-cli/src/service/mod.rs | 4 +- .../src/chainhooks/stacks/mod.rs | 37 ++++++++----------- components/chainhook-sdk/src/indexer/mod.rs | 13 +++++-- components/chainhook-types-rs/src/rosetta.rs | 14 +++++-- 4 files changed, 36 insertions(+), 32 deletions(-) diff --git a/components/chainhook-cli/src/service/mod.rs b/components/chainhook-cli/src/service/mod.rs index a994b3365..d93fcf335 100644 --- a/components/chainhook-cli/src/service/mod.rs +++ b/components/chainhook-cli/src/service/mod.rs @@ -548,7 +548,7 @@ impl Service { } StacksChainEvent::ChainUpdatedWithMicroblocks(_) | StacksChainEvent::ChainUpdatedWithMicroblocksReorg(_) => {}, - StacksChainEvent::ChainUpdatedWithStackerDbChunks(data) => { + StacksChainEvent::ChainUpdatedWithNonConsensusEvents(_data) => { // TODO(rafaelcr): Store signer data. } }, @@ -619,7 +619,7 @@ impl Service { } StacksChainEvent::ChainUpdatedWithMicroblocks(_) | StacksChainEvent::ChainUpdatedWithMicroblocksReorg(_) => {}, - StacksChainEvent::ChainUpdatedWithStackerDbChunks(_) => { + StacksChainEvent::ChainUpdatedWithNonConsensusEvents(_) => { // TODO(rafaelcr): Send via HTTP payload. }, }; diff --git a/components/chainhook-sdk/src/chainhooks/stacks/mod.rs b/components/chainhook-sdk/src/chainhooks/stacks/mod.rs index 5df74c567..f51a3cde8 100644 --- a/components/chainhook-sdk/src/chainhooks/stacks/mod.rs +++ b/components/chainhook-sdk/src/chainhooks/stacks/mod.rs @@ -9,9 +9,9 @@ use super::types::{ HookAction, }; use chainhook_types::{ - BlockIdentifier, StacksChainEvent, StacksNetwork, StacksStackerDbChunk, StacksTransactionData, - StacksTransactionEvent, StacksTransactionEventPayload, StacksTransactionKind, - TransactionIdentifier, + BlockIdentifier, StacksChainEvent, StacksNetwork, StacksNonConsensusEventData, + StacksTransactionData, StacksTransactionEvent, StacksTransactionEventPayload, + StacksTransactionKind, TransactionIdentifier, }; use clarity::codec::StacksMessageCodec; use clarity::vm::types::{ @@ -478,7 +478,7 @@ pub struct StacksTriggerChainhook<'a> { pub chainhook: &'a StacksChainhookInstance, pub apply: Vec<(Vec<&'a StacksTransactionData>, &'a dyn AbstractStacksBlock)>, pub rollback: Vec<(Vec<&'a StacksTransactionData>, &'a dyn AbstractStacksBlock)>, - pub events: Vec<&'a dyn AbstractStacksNonConsensusEvent>, + pub events: Vec<&'a StacksNonConsensusEventData>, } #[derive(Clone, Debug)] @@ -493,12 +493,6 @@ pub struct StacksRollbackTransactionPayload { pub transactions: Vec, } -#[derive(Clone, Debug)] -pub struct StacksNonConsensusEventPayload { - pub block_identifier: BlockIdentifier, - pub transactions: Vec, -} - #[derive(Clone, Debug)] pub struct StacksChainhookPayload { pub uuid: String, @@ -508,7 +502,7 @@ pub struct StacksChainhookPayload { pub struct StacksChainhookOccurrencePayload { pub apply: Vec, pub rollback: Vec, - pub events: Vec, + pub events: Vec, pub chainhook: StacksChainhookPayload, } @@ -540,6 +534,7 @@ impl StacksChainhookOccurrencePayload { chainhook: StacksChainhookPayload { uuid: trigger.chainhook.uuid.clone(), }, + events: trigger.events.into_iter().cloned().collect::>(), } } } @@ -749,17 +744,14 @@ pub fn evaluate_stacks_chainhooks_on_chain_event<'a>( } } } - StacksChainEvent::ChainUpdatedWithStackerDbChunks(data) => { + StacksChainEvent::ChainUpdatedWithNonConsensusEvents(data) => { + #[cfg(feature = "stacks-signers")] for chainhook in active_chainhooks.iter() { let mut events = vec![]; evaluated_predicates.insert(chainhook.uuid.as_str(), &data.received_at_block); - let mut chunks: Vec<&dyn AbstractStacksNonConsensusEvent> = vec![]; - for chunk in data.chunks.iter() { - chunks.push(chunk); - } let (mut occurrences, mut expirations) = - evaluate_stacks_predicate_on_stackerdb_chunks(chunks, chainhook, ctx); + evaluate_stacks_predicate_on_stackerdb_chunks(&data.events, chainhook, ctx); events.append(&mut occurrences); expired_predicates.append(&mut expirations); @@ -850,22 +842,23 @@ pub fn evaluate_stacks_predicate_on_block<'a>( #[cfg(feature = "stacks-signers")] pub fn evaluate_stacks_predicate_on_stackerdb_chunks<'a>( - chunks: Vec<&'a dyn AbstractStacksNonConsensusEvent>, + events: &'a Vec, chainhook: &'a StacksChainhookInstance, _ctx: &Context, ) -> ( - Vec<&'a dyn AbstractStacksNonConsensusEvent>, + Vec<&'a StacksNonConsensusEventData>, BTreeMap<&'a str, &'a BlockIdentifier>, ) { let mut occurrences = vec![]; let expired_predicates = BTreeMap::new(); - for chunk in chunks { + for event in events { match &chainhook.predicate { StacksPredicate::SignerMessage(StacksSignerMessagePredicate::AfterTimestamp( timestamp, )) => { + let StacksNonConsensusEventData::SignerMessage(chunk) = event; if chunk.get_timestamp() >= *timestamp as i64 { - occurrences.push(chunk); + occurrences.push(event); } } StacksPredicate::SignerMessage(StacksSignerMessagePredicate::FromSignerPubKey(_)) => { @@ -905,7 +898,7 @@ pub fn evaluate_stacks_predicate_on_transaction<'a>( _ => false, }, StacksPredicate::ContractDeployment(StacksContractDeploymentPredicate::ImplementTrait( - stacks_trait, + _stacks_trait, )) => match &transaction.metadata.kind { StacksTransactionKind::ContractDeployment(_actual_deployment) => { ctx.try_log(|logger| { diff --git a/components/chainhook-sdk/src/indexer/mod.rs b/components/chainhook-sdk/src/indexer/mod.rs index 9de41545b..bffc030b5 100644 --- a/components/chainhook-sdk/src/indexer/mod.rs +++ b/components/chainhook-sdk/src/indexer/mod.rs @@ -174,7 +174,9 @@ impl Indexer { receipt_time: u64, ctx: &Context, ) -> Result, String> { - use chainhook_types::StacksChainUpdatedWithStackerDbChunksData; + use chainhook_types::{ + StacksChainUpdatedWithNonConsensusEventsData, StacksNonConsensusEventData, + }; let Some(chain_tip) = self.stacks_blocks_pool.get_canonical_fork_chain_tip() else { return Err("StackerDB chunk received with no canonical chain tip".to_string()); }; @@ -185,9 +187,12 @@ impl Indexer { ctx, )?; if chunks.len() > 0 { - Ok(Some(StacksChainEvent::ChainUpdatedWithStackerDbChunks( - StacksChainUpdatedWithStackerDbChunksData { - chunks, + Ok(Some(StacksChainEvent::ChainUpdatedWithNonConsensusEvents( + StacksChainUpdatedWithNonConsensusEventsData { + events: chunks + .into_iter() + .map(|chunk| StacksNonConsensusEventData::SignerMessage(chunk)) + .collect(), received_at: receipt_time, received_at_block: chain_tip.clone(), }, diff --git a/components/chainhook-types-rs/src/rosetta.rs b/components/chainhook-types-rs/src/rosetta.rs index d5dba43bd..8597f527c 100644 --- a/components/chainhook-types-rs/src/rosetta.rs +++ b/components/chainhook-types-rs/src/rosetta.rs @@ -664,6 +664,12 @@ pub struct BlockchainUpdatedWithReorg { pub confirmed_headers: Vec, } +#[derive(Clone, Debug, PartialEq, Serialize)] +#[serde(tag = "type", content = "data")] +pub enum StacksNonConsensusEventData { + SignerMessage(StacksStackerDbChunk), +} + #[derive(Debug, Clone, PartialEq, Serialize)] pub struct BlockHeader { pub block_identifier: BlockIdentifier, @@ -691,8 +697,8 @@ pub struct BitcoinChainUpdatedWithReorgData { } #[derive(Debug, Clone, PartialEq, Serialize)] -pub struct StacksChainUpdatedWithStackerDbChunksData { - pub chunks: Vec, +pub struct StacksChainUpdatedWithNonConsensusEventsData { + pub events: Vec, pub received_at: u64, pub received_at_block: BlockIdentifier, } @@ -704,7 +710,7 @@ pub enum StacksChainEvent { ChainUpdatedWithReorg(StacksChainUpdatedWithReorgData), ChainUpdatedWithMicroblocks(StacksChainUpdatedWithMicroblocksData), ChainUpdatedWithMicroblocksReorg(StacksChainUpdatedWithMicroblocksReorgData), - ChainUpdatedWithStackerDbChunks(StacksChainUpdatedWithStackerDbChunksData), + ChainUpdatedWithNonConsensusEvents(StacksChainUpdatedWithNonConsensusEventsData), } impl StacksChainEvent { @@ -734,7 +740,7 @@ impl StacksChainEvent { .microblocks_to_apply .first() .and_then(|b| Some(&b.metadata.anchor_block_identifier)), - StacksChainEvent::ChainUpdatedWithStackerDbChunks(_) => None, + StacksChainEvent::ChainUpdatedWithNonConsensusEvents(_) => None, } } } From 41c7decc68db979c6ca50562a7e3be33e89e1aa5 Mon Sep 17 00:00:00 2001 From: Rafael Cardenas Date: Tue, 22 Oct 2024 11:32:55 -0600 Subject: [PATCH 05/15] style: fix --- .../src/chainhooks/stacks/mod.rs | 28 +++++++++---------- .../chainhook-sdk/src/indexer/stacks/mod.rs | 10 +++---- components/chainhook-sdk/src/observer/mod.rs | 4 +-- 3 files changed, 20 insertions(+), 22 deletions(-) diff --git a/components/chainhook-sdk/src/chainhooks/stacks/mod.rs b/components/chainhook-sdk/src/chainhooks/stacks/mod.rs index f51a3cde8..997f5a2cb 100644 --- a/components/chainhook-sdk/src/chainhooks/stacks/mod.rs +++ b/components/chainhook-sdk/src/chainhooks/stacks/mod.rs @@ -1,7 +1,5 @@ use crate::observer::EventObserverConfig; -use crate::utils::{ - AbstractStacksBlock, AbstractStacksNonConsensusEvent, Context, MAX_BLOCK_HEIGHTS_ENTRIES, -}; +use crate::utils::{AbstractStacksBlock, Context, MAX_BLOCK_HEIGHTS_ENTRIES}; use super::types::validate_txid; use super::types::{ @@ -316,7 +314,7 @@ impl StacksPredicate { // TODO(rafaelcr): Validate pubkey format } #[cfg(feature = "stacks-signers")] - StacksPredicate::SignerMessage(_) => {} + StacksPredicate::SignerMessage(StacksSignerMessagePredicate::AfterTimestamp(_)) => {} } Ok(()) } @@ -744,27 +742,25 @@ pub fn evaluate_stacks_chainhooks_on_chain_event<'a>( } } } + #[cfg(feature = "stacks-signers")] StacksChainEvent::ChainUpdatedWithNonConsensusEvents(data) => { - #[cfg(feature = "stacks-signers")] for chainhook in active_chainhooks.iter() { - let mut events = vec![]; - evaluated_predicates.insert(chainhook.uuid.as_str(), &data.received_at_block); - let (mut occurrences, mut expirations) = - evaluate_stacks_predicate_on_stackerdb_chunks(&data.events, chainhook, ctx); - events.append(&mut occurrences); + let (occurrences, mut expirations) = + evaluate_stacks_predicate_on_non_consensus_events(&data.events, chainhook, ctx); expired_predicates.append(&mut expirations); - - if events.len() > 0 { + if occurrences.len() > 0 { triggered_predicates.push(StacksTriggerChainhook { chainhook, apply: vec![], rollback: vec![], - events, + events: occurrences, }); } } } + #[cfg(not(feature = "stacks-signers"))] + StacksChainEvent::ChainUpdatedWithNonConsensusEvents(_) => {} } ( triggered_predicates, @@ -841,7 +837,7 @@ pub fn evaluate_stacks_predicate_on_block<'a>( } #[cfg(feature = "stacks-signers")] -pub fn evaluate_stacks_predicate_on_stackerdb_chunks<'a>( +pub fn evaluate_stacks_predicate_on_non_consensus_events<'a>( events: &'a Vec, chainhook: &'a StacksChainhookInstance, _ctx: &Context, @@ -849,6 +845,8 @@ pub fn evaluate_stacks_predicate_on_stackerdb_chunks<'a>( Vec<&'a StacksNonConsensusEventData>, BTreeMap<&'a str, &'a BlockIdentifier>, ) { + use crate::utils::AbstractStacksNonConsensusEvent; + let mut occurrences = vec![]; let expired_predicates = BTreeMap::new(); for event in events { @@ -862,7 +860,7 @@ pub fn evaluate_stacks_predicate_on_stackerdb_chunks<'a>( } } StacksPredicate::SignerMessage(StacksSignerMessagePredicate::FromSignerPubKey(_)) => { - todo!() + // TODO(rafaelcr): Evaluate on pubkey } StacksPredicate::BlockHeight(_) | StacksPredicate::ContractDeployment(_) diff --git a/components/chainhook-sdk/src/indexer/stacks/mod.rs b/components/chainhook-sdk/src/indexer/stacks/mod.rs index 4a7ef343d..d969c5130 100644 --- a/components/chainhook-sdk/src/indexer/stacks/mod.rs +++ b/components/chainhook-sdk/src/indexer/stacks/mod.rs @@ -12,7 +12,7 @@ use clarity::vm::types::{SequenceData, Value as ClarityValue}; use hiro_system_kit::slog; use rocket::serde::json::Value as JsonValue; use rocket::serde::Deserialize; -use stacks_codec::codec::{NakamotoBlock, StacksTransaction, TransactionAuth, TransactionPayload}; +use stacks_codec::codec::{StacksTransaction, TransactionAuth, TransactionPayload}; use std::collections::{BTreeMap, HashMap, HashSet}; use std::convert::TryInto; use std::io::Cursor; @@ -38,10 +38,10 @@ pub struct NewBlock { #[serde(skip_serializing_if = "Option::is_none")] pub block_time: Option, - + #[serde(skip_serializing_if = "Option::is_none")] pub signer_bitvec: Option, - + #[serde(skip_serializing_if = "Option::is_none")] pub signer_signature: Option>, @@ -761,7 +761,7 @@ pub fn standardize_stacks_stackerdb_chunks( pubkey: get_signer_pubkey_from_stackerdb_chunk_slot(slot, &data_bytes)?, message, received_at: receipt_time, - received_at_block: chain_tip.clone() + received_at_block: chain_tip.clone(), }); } @@ -769,7 +769,7 @@ pub fn standardize_stacks_stackerdb_chunks( } #[cfg(feature = "stacks-signers")] -pub fn standardize_stacks_nakamoto_block(block: &NakamotoBlock) -> NakamotoBlockData { +pub fn standardize_stacks_nakamoto_block(block: &stacks_codec::codec::NakamotoBlock) -> NakamotoBlockData { use miniscript::bitcoin::hex::Case; use miniscript::bitcoin::hex::DisplayHex; diff --git a/components/chainhook-sdk/src/observer/mod.rs b/components/chainhook-sdk/src/observer/mod.rs index 0e9a15cd6..22bf564f5 100644 --- a/components/chainhook-sdk/src/observer/mod.rs +++ b/components/chainhook-sdk/src/observer/mod.rs @@ -885,7 +885,7 @@ pub fn start_event_observer( pub async fn start_bitcoin_event_observer( config: EventObserverConfig, - observer_commands_tx: Sender, + _observer_commands_tx: Sender, observer_commands_rx: Receiver, observer_events_tx: Option>, observer_sidecar: Option, @@ -897,7 +897,7 @@ pub async fn start_bitcoin_event_observer( let ctx_moved = ctx.clone(); let config_moved = config.clone(); let _ = hiro_system_kit::thread_named("ZMQ handler").spawn(move || { - let future = zmq::start_zeromq_runloop(&config_moved, observer_commands_tx, &ctx_moved); + let future = zmq::start_zeromq_runloop(&config_moved, _observer_commands_tx, &ctx_moved); hiro_system_kit::nestable_block_on(future); }); } From c211f086468fafc92a15a6f41965def0b5a94f9b Mon Sep 17 00:00:00 2001 From: Rafael Cardenas Date: Wed, 23 Oct 2024 07:36:08 -0600 Subject: [PATCH 06/15] fix: parsing --- components/chainhook-cli/src/scan/stacks.rs | 1 + components/chainhook-cli/src/service/mod.rs | 5 +++-- components/chainhook-cli/src/storage/mod.rs | 2 ++ .../chainhook-sdk/src/indexer/stacks/mod.rs | 15 ++++++++------- .../src/indexer/tests/helpers/stacks_events.rs | 15 +++++++++------ components/chainhook-sdk/src/observer/http.rs | 2 +- 6 files changed, 24 insertions(+), 16 deletions(-) diff --git a/components/chainhook-cli/src/scan/stacks.rs b/components/chainhook-cli/src/scan/stacks.rs index eda01a0a2..9e021513e 100644 --- a/components/chainhook-cli/src/scan/stacks.rs +++ b/components/chainhook-cli/src/scan/stacks.rs @@ -339,6 +339,7 @@ pub async fn scan_stacks_chainstate_via_rocksdb_using_predicate( chainhook: predicate_spec, apply: hits_per_blocks, rollback: vec![], + // TODO(rafaelcr): Query for non consensus events which fall between block timestamps to fill in here events: vec![] }; let res = match handle_stacks_hook_action( diff --git a/components/chainhook-cli/src/service/mod.rs b/components/chainhook-cli/src/service/mod.rs index d93fcf335..7e50be6d3 100644 --- a/components/chainhook-cli/src/service/mod.rs +++ b/components/chainhook-cli/src/service/mod.rs @@ -548,8 +548,9 @@ impl Service { } StacksChainEvent::ChainUpdatedWithMicroblocks(_) | StacksChainEvent::ChainUpdatedWithMicroblocksReorg(_) => {}, - StacksChainEvent::ChainUpdatedWithNonConsensusEvents(_data) => { + StacksChainEvent::ChainUpdatedWithNonConsensusEvents(data) => { // TODO(rafaelcr): Store signer data. + println!("signer message: {:?}", data); } }, Err(e) => { @@ -620,7 +621,7 @@ impl Service { StacksChainEvent::ChainUpdatedWithMicroblocks(_) | StacksChainEvent::ChainUpdatedWithMicroblocksReorg(_) => {}, StacksChainEvent::ChainUpdatedWithNonConsensusEvents(_) => { - // TODO(rafaelcr): Send via HTTP payload. + // TODO(rafaelcr): Expire signer message predicates when appropriate }, }; update_status_from_report( diff --git a/components/chainhook-cli/src/storage/mod.rs b/components/chainhook-cli/src/storage/mod.rs index 55fa9e8f9..820fbe7dd 100644 --- a/components/chainhook-cli/src/storage/mod.rs +++ b/components/chainhook-cli/src/storage/mod.rs @@ -1,3 +1,5 @@ +pub mod signers; + use std::collections::VecDeque; use std::path::PathBuf; diff --git a/components/chainhook-sdk/src/indexer/stacks/mod.rs b/components/chainhook-sdk/src/indexer/stacks/mod.rs index d969c5130..8de21e9b7 100644 --- a/components/chainhook-sdk/src/indexer/stacks/mod.rs +++ b/components/chainhook-sdk/src/indexer/stacks/mod.rs @@ -305,16 +305,17 @@ pub struct ContractReadonlyCall { #[cfg(feature = "stacks-signers")] #[derive(Deserialize, Debug)] -pub struct NewStackerDbChunkIssuer { - pub issuer_id: u32, - pub slots: Vec, -} +pub struct NewStackerDbChunkIssuerId(pub u32); + +#[cfg(feature = "stacks-signers")] +#[derive(Deserialize, Debug)] +pub struct NewStackerDbChunkIssuerSlots(pub Vec); #[cfg(feature = "stacks-signers")] #[derive(Deserialize, Debug)] pub struct NewStackerDbChunksContractId { pub name: String, - pub issuer: Vec, + pub issuer: (NewStackerDbChunkIssuerId, NewStackerDbChunkIssuerSlots), } #[cfg(feature = "stacks-signers")] @@ -323,7 +324,7 @@ pub struct NewSignerModifiedSlot { pub sig: String, pub data: String, pub slot_id: u64, - pub version: u64, + pub slot_version: u64, } #[cfg(feature = "stacks-signers")] @@ -816,7 +817,7 @@ pub fn get_signer_pubkey_from_stackerdb_chunk_slot( }; let mut digest_bytes = slot.slot_id.to_be_bytes().to_vec(); - digest_bytes.extend(slot.version.to_be_bytes().to_vec()); + digest_bytes.extend(slot.slot_version.to_be_bytes().to_vec()); let data_bytes_hashed = Sha512Trunc256Sum::from_data(&data_bytes).to_bytes(); digest_bytes.extend(data_bytes_hashed); let digest = Sha512Trunc256Sum::from_data(&digest_bytes).to_bytes(); diff --git a/components/chainhook-sdk/src/indexer/tests/helpers/stacks_events.rs b/components/chainhook-sdk/src/indexer/tests/helpers/stacks_events.rs index e944b986f..6f8386861 100644 --- a/components/chainhook-sdk/src/indexer/tests/helpers/stacks_events.rs +++ b/components/chainhook-sdk/src/indexer/tests/helpers/stacks_events.rs @@ -127,21 +127,24 @@ pub fn create_new_stackerdb_chunk( slot_data: String, ) -> crate::indexer::stacks::NewStackerDbChunks { use crate::indexer::stacks::{ - NewSignerModifiedSlot, NewStackerDbChunkIssuer, NewStackerDbChunksContractId, + NewSignerModifiedSlot, NewStackerDbChunkIssuerId, NewStackerDbChunkIssuerSlots, + NewStackerDbChunksContractId, }; crate::indexer::stacks::NewStackerDbChunks { contract_id: NewStackerDbChunksContractId { name: contract_name, - issuer: vec![NewStackerDbChunkIssuer { - issuer_id: 26, - slots: vec![0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - }], + issuer: ( + NewStackerDbChunkIssuerId(26), + NewStackerDbChunkIssuerSlots(vec![ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]), + ), }, modified_slots: vec![NewSignerModifiedSlot { sig: slot_sig, data: slot_data, slot_id: 1, - version: 141, + slot_version: 141, }], } } diff --git a/components/chainhook-sdk/src/observer/http.rs b/components/chainhook-sdk/src/observer/http.rs index 7468fa4a1..154eac688 100644 --- a/components/chainhook-sdk/src/observer/http.rs +++ b/components/chainhook-sdk/src/observer/http.rs @@ -178,8 +178,8 @@ pub fn handle_new_stacks_block( success_response() } -#[cfg(feature = "stacks-signers")] #[post("/stackerdb_chunks", format = "application/json", data = "")] +#[cfg(feature = "stacks-signers")] pub fn handle_stackerdb_chunks( indexer_rw_lock: &State>>, payload: Json, From 0a8002c4d4bf3afd9a0c4c0b008e6c0be85ebc45 Mon Sep 17 00:00:00 2001 From: Rafael Cardenas Date: Wed, 23 Oct 2024 11:06:03 -0600 Subject: [PATCH 07/15] fix: streaming works --- components/chainhook-cli/src/storage/mod.rs | 2 - .../src/chainhooks/stacks/mod.rs | 58 +++++++++++++------ components/chainhook-sdk/src/indexer/mod.rs | 11 ++-- .../chainhook-sdk/src/indexer/stacks/mod.rs | 12 ++-- .../chainhook-sdk/src/indexer/stacks/tests.rs | 12 +--- components/chainhook-sdk/src/observer/mod.rs | 15 ++++- components/chainhook-sdk/src/utils/mod.rs | 19 +----- components/chainhook-types-rs/src/rosetta.rs | 11 +++- components/chainhook-types-rs/src/signers.rs | 2 - .../typescript/src/schemas/stacks/payload.ts | 6 +- .../typescript/src/schemas/stacks/signers.ts | 18 +++--- 11 files changed, 87 insertions(+), 79 deletions(-) diff --git a/components/chainhook-cli/src/storage/mod.rs b/components/chainhook-cli/src/storage/mod.rs index 820fbe7dd..55fa9e8f9 100644 --- a/components/chainhook-cli/src/storage/mod.rs +++ b/components/chainhook-cli/src/storage/mod.rs @@ -1,5 +1,3 @@ -pub mod signers; - use std::collections::VecDeque; use std::path::PathBuf; diff --git a/components/chainhook-sdk/src/chainhooks/stacks/mod.rs b/components/chainhook-sdk/src/chainhooks/stacks/mod.rs index 997f5a2cb..d1cf96ab2 100644 --- a/components/chainhook-sdk/src/chainhooks/stacks/mod.rs +++ b/components/chainhook-sdk/src/chainhooks/stacks/mod.rs @@ -744,18 +744,25 @@ pub fn evaluate_stacks_chainhooks_on_chain_event<'a>( } #[cfg(feature = "stacks-signers")] StacksChainEvent::ChainUpdatedWithNonConsensusEvents(data) => { - for chainhook in active_chainhooks.iter() { - evaluated_predicates.insert(chainhook.uuid.as_str(), &data.received_at_block); - let (occurrences, mut expirations) = - evaluate_stacks_predicate_on_non_consensus_events(&data.events, chainhook, ctx); - expired_predicates.append(&mut expirations); - if occurrences.len() > 0 { - triggered_predicates.push(StacksTriggerChainhook { - chainhook, - apply: vec![], - rollback: vec![], - events: occurrences, - }); + if let Some(first_event) = data.events.first() { + for chainhook in active_chainhooks.iter() { + evaluated_predicates + .insert(chainhook.uuid.as_str(), &first_event.received_at_block); + let (occurrences, mut expirations) = + evaluate_stacks_predicate_on_non_consensus_events( + &data.events, + chainhook, + ctx, + ); + expired_predicates.append(&mut expirations); + if occurrences.len() > 0 { + triggered_predicates.push(StacksTriggerChainhook { + chainhook, + apply: vec![], + rollback: vec![], + events: occurrences, + }); + } } } } @@ -832,7 +839,7 @@ pub fn evaluate_stacks_predicate_on_block<'a>( | StacksPredicate::PrintEvent(_) | StacksPredicate::Txid(_) => unreachable!(), #[cfg(feature = "stacks-signers")] - StacksPredicate::SignerMessage(_) => unreachable!(), + StacksPredicate::SignerMessage(_) => false, } } @@ -845,8 +852,6 @@ pub fn evaluate_stacks_predicate_on_non_consensus_events<'a>( Vec<&'a StacksNonConsensusEventData>, BTreeMap<&'a str, &'a BlockIdentifier>, ) { - use crate::utils::AbstractStacksNonConsensusEvent; - let mut occurrences = vec![]; let expired_predicates = BTreeMap::new(); for event in events { @@ -854,8 +859,7 @@ pub fn evaluate_stacks_predicate_on_non_consensus_events<'a>( StacksPredicate::SignerMessage(StacksSignerMessagePredicate::AfterTimestamp( timestamp, )) => { - let StacksNonConsensusEventData::SignerMessage(chunk) = event; - if chunk.get_timestamp() >= *timestamp as i64 { + if event.received_at >= *timestamp { occurrences.push(event); } } @@ -1086,10 +1090,27 @@ pub fn evaluate_stacks_predicate_on_transaction<'a>( } StacksPredicate::BlockHeight(_) => unreachable!(), #[cfg(feature = "stacks-signers")] - StacksPredicate::SignerMessage(_) => unreachable!(), + StacksPredicate::SignerMessage(_) => false, } } +#[cfg(feature = "stacks-signers")] +fn serialize_stacks_non_consensus_event( + event: &StacksNonConsensusEventData, + _ctx: &Context, +) -> serde_json::Value { + use chainhook_types::StacksNonConsensusEventPayloadData; + + let payload = match &event.payload { + StacksNonConsensusEventPayloadData::SignerMessage(chunk) => chunk, + }; + json!({ + "payload": payload, + "received_at": event.received_at, + "received_at_block": event.received_at_block, + }) +} + fn serialize_stacks_block( block: &dyn AbstractStacksBlock, transactions: Vec<&StacksTransactionData>, @@ -1400,6 +1421,7 @@ pub fn serialize_stacks_payload_to_json<'a>( "rollback": trigger.rollback.into_iter().map(|(transactions, block)| { serialize_stacks_block(block, transactions, decode_clarity_values, include_contract_abi, ctx) }).collect::>(), + "events": trigger.events.into_iter().map(|event| serialize_stacks_non_consensus_event(event, ctx)).collect::>(), "chainhook": { "uuid": trigger.chainhook.uuid, "predicate": trigger.chainhook.predicate, diff --git a/components/chainhook-sdk/src/indexer/mod.rs b/components/chainhook-sdk/src/indexer/mod.rs index bffc030b5..9aaea364f 100644 --- a/components/chainhook-sdk/src/indexer/mod.rs +++ b/components/chainhook-sdk/src/indexer/mod.rs @@ -176,14 +176,13 @@ impl Indexer { ) -> Result, String> { use chainhook_types::{ StacksChainUpdatedWithNonConsensusEventsData, StacksNonConsensusEventData, + StacksNonConsensusEventPayloadData, }; let Some(chain_tip) = self.stacks_blocks_pool.get_canonical_fork_chain_tip() else { return Err("StackerDB chunk received with no canonical chain tip".to_string()); }; let chunks = stacks::standardize_stacks_marshalled_stackerdb_chunks( marshalled_stackerdb_chunks, - receipt_time, - chain_tip, ctx, )?; if chunks.len() > 0 { @@ -191,10 +190,12 @@ impl Indexer { StacksChainUpdatedWithNonConsensusEventsData { events: chunks .into_iter() - .map(|chunk| StacksNonConsensusEventData::SignerMessage(chunk)) + .map(|chunk| StacksNonConsensusEventData { + payload: StacksNonConsensusEventPayloadData::SignerMessage(chunk), + received_at: receipt_time, + received_at_block: chain_tip.clone(), + }) .collect(), - received_at: receipt_time, - received_at_block: chain_tip.clone(), }, ))) } else { diff --git a/components/chainhook-sdk/src/indexer/stacks/mod.rs b/components/chainhook-sdk/src/indexer/stacks/mod.rs index 8de21e9b7..1c9308dc8 100644 --- a/components/chainhook-sdk/src/indexer/stacks/mod.rs +++ b/components/chainhook-sdk/src/indexer/stacks/mod.rs @@ -648,21 +648,17 @@ pub fn standardize_stacks_microblock_trail( #[cfg(feature = "stacks-signers")] pub fn standardize_stacks_marshalled_stackerdb_chunks( marshalled_stackerdb_chunks: JsonValue, - receipt_time: u64, - chain_tip: &BlockIdentifier, _ctx: &Context, ) -> Result, String> { let mut stackerdb_chunks: NewStackerDbChunks = serde_json::from_value(marshalled_stackerdb_chunks) .map_err(|e| format!("unable to parse stackerdb chunks {e}"))?; - standardize_stacks_stackerdb_chunks(&mut stackerdb_chunks, receipt_time, chain_tip) + standardize_stacks_stackerdb_chunks(&mut stackerdb_chunks) } #[cfg(feature = "stacks-signers")] pub fn standardize_stacks_stackerdb_chunks( stackerdb_chunks: &NewStackerDbChunks, - receipt_time: u64, - chain_tip: &BlockIdentifier, ) -> Result, String> { use stacks_codec::codec::BlockResponse; use stacks_codec::codec::RejectCode; @@ -761,8 +757,6 @@ pub fn standardize_stacks_stackerdb_chunks( sig: slot.sig.clone(), pubkey: get_signer_pubkey_from_stackerdb_chunk_slot(slot, &data_bytes)?, message, - received_at: receipt_time, - received_at_block: chain_tip.clone(), }); } @@ -770,7 +764,9 @@ pub fn standardize_stacks_stackerdb_chunks( } #[cfg(feature = "stacks-signers")] -pub fn standardize_stacks_nakamoto_block(block: &stacks_codec::codec::NakamotoBlock) -> NakamotoBlockData { +pub fn standardize_stacks_nakamoto_block( + block: &stacks_codec::codec::NakamotoBlock, +) -> NakamotoBlockData { use miniscript::bitcoin::hex::Case; use miniscript::bitcoin::hex::DisplayHex; diff --git a/components/chainhook-sdk/src/indexer/stacks/tests.rs b/components/chainhook-sdk/src/indexer/stacks/tests.rs index b129121bc..b26a284be 100644 --- a/components/chainhook-sdk/src/indexer/stacks/tests.rs +++ b/components/chainhook-sdk/src/indexer/stacks/tests.rs @@ -402,7 +402,7 @@ fn into_chainhook_event_rejects_invalid_missing_event() { #[test] #[cfg(feature = "stacks-signers")] fn stackerdb_chunks_covert_into_signer_messages() { - use chainhook_types::{BlockIdentifier, BlockResponseData, StacksSignerMessage}; + use chainhook_types::{BlockResponseData, StacksSignerMessage}; use crate::indexer::tests::helpers::stacks_events::create_new_stackerdb_chunk; @@ -413,15 +413,7 @@ fn stackerdb_chunks_covert_into_signer_messages() { "01fc3c06f6e0ae5b13c9bb53763661817e55c8e7f1ecab8b4d4b65b283d2dd39f0099e3ea1e25e765f4f0e1dfb0a432309a16a2ec10940e1a14cb9e9b1cbf27edc".to_string(), "010074aff146904763a787aa14c614d0dd1fc63b537bdb2fd351cdf881f6db75f986005eb55250597b25acbf99d3dd3c2fa8189046e1b5d21309a44cbaf2b327c09b0159a01ed3f0094bfa9e5f72f5d894e12ce252081eab5396eb8bba137bddfc365b".to_string() ); - let parsed_chunk = standardize_stacks_stackerdb_chunks( - &new_chunks, - 1729013425, - &BlockIdentifier { - index: 170355, - hash: "0x519df2ad0d86a62dc078865486a1f3dbb2f1a8934da81b679561738820964fe0".to_string(), - }, - ) - .unwrap(); + let parsed_chunk = standardize_stacks_stackerdb_chunks(&new_chunks).unwrap(); assert_eq!(parsed_chunk.len(), 1); let message = &parsed_chunk[0]; diff --git a/components/chainhook-sdk/src/observer/mod.rs b/components/chainhook-sdk/src/observer/mod.rs index 22bf564f5..038642b4d 100644 --- a/components/chainhook-sdk/src/observer/mod.rs +++ b/components/chainhook-sdk/src/observer/mod.rs @@ -897,7 +897,8 @@ pub async fn start_bitcoin_event_observer( let ctx_moved = ctx.clone(); let config_moved = config.clone(); let _ = hiro_system_kit::thread_named("ZMQ handler").spawn(move || { - let future = zmq::start_zeromq_runloop(&config_moved, _observer_commands_tx, &ctx_moved); + let future = + zmq::start_zeromq_runloop(&config_moved, _observer_commands_tx, &ctx_moved); hiro_system_kit::nestable_block_on(future); }); } @@ -1658,12 +1659,20 @@ pub async fn start_observer_commands_handler( report.track_expiration(uuid, block_identifier); } for entry in predicates_triggered.iter() { - let blocks_ids = entry + let mut block_ids = entry .apply .iter() .map(|e| e.1.get_identifier()) .collect::>(); - report.track_trigger(&entry.chainhook.uuid, &blocks_ids); + let mut event_block_ids = entry + .events + .iter() + .map(|e| &e.received_at_block) + .collect::>(); + if event_block_ids.len() > 0 { + block_ids.append(&mut event_block_ids); + } + report.track_trigger(&entry.chainhook.uuid, &block_ids); } ctx.try_log(|logger| { slog::info!( diff --git a/components/chainhook-sdk/src/utils/mod.rs b/components/chainhook-sdk/src/utils/mod.rs index f2de01d3e..b74f2edbe 100644 --- a/components/chainhook-sdk/src/utils/mod.rs +++ b/components/chainhook-sdk/src/utils/mod.rs @@ -6,7 +6,7 @@ use std::{ }; use chainhook_types::{ - BitcoinBlockData, BlockHeader, BlockIdentifier, StacksBlockData, StacksMicroblockData, StacksStackerDbChunk, StacksTransactionData + BitcoinBlockData, BlockHeader, BlockIdentifier, StacksBlockData, StacksMicroblockData, StacksTransactionData }; use hiro_system_kit::slog::{self, Logger}; use reqwest::RequestBuilder; @@ -92,23 +92,6 @@ impl AbstractStacksBlock for StacksMicroblockData { } } -/// Trait for Stacks events that are not part of the blockchain consensus but are otherwise broadcasted by Stacks nodes to event -/// listeners. -pub trait AbstractStacksNonConsensusEvent { - fn get_timestamp(&self) -> i64; - fn get_received_at_block(&self) -> BlockIdentifier; -} - -impl AbstractStacksNonConsensusEvent for StacksStackerDbChunk { - fn get_timestamp(&self) -> i64 { - self.received_at as i64 - } - - fn get_received_at_block(&self) -> BlockIdentifier { - self.received_at_block.clone() - } -} - pub trait AbstractBlock { fn get_identifier(&self) -> &BlockIdentifier; fn get_parent_identifier(&self) -> &BlockIdentifier; diff --git a/components/chainhook-types-rs/src/rosetta.rs b/components/chainhook-types-rs/src/rosetta.rs index 8597f527c..8507a70c6 100644 --- a/components/chainhook-types-rs/src/rosetta.rs +++ b/components/chainhook-types-rs/src/rosetta.rs @@ -666,10 +666,17 @@ pub struct BlockchainUpdatedWithReorg { #[derive(Clone, Debug, PartialEq, Serialize)] #[serde(tag = "type", content = "data")] -pub enum StacksNonConsensusEventData { +pub enum StacksNonConsensusEventPayloadData { SignerMessage(StacksStackerDbChunk), } +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct StacksNonConsensusEventData { + pub payload: StacksNonConsensusEventPayloadData, + pub received_at: u64, + pub received_at_block: BlockIdentifier, +} + #[derive(Debug, Clone, PartialEq, Serialize)] pub struct BlockHeader { pub block_identifier: BlockIdentifier, @@ -699,8 +706,6 @@ pub struct BitcoinChainUpdatedWithReorgData { #[derive(Debug, Clone, PartialEq, Serialize)] pub struct StacksChainUpdatedWithNonConsensusEventsData { pub events: Vec, - pub received_at: u64, - pub received_at_block: BlockIdentifier, } #[allow(dead_code)] diff --git a/components/chainhook-types-rs/src/signers.rs b/components/chainhook-types-rs/src/signers.rs index 7fb2efb4a..3fcd15cb3 100644 --- a/components/chainhook-types-rs/src/signers.rs +++ b/components/chainhook-types-rs/src/signers.rs @@ -93,6 +93,4 @@ pub struct StacksStackerDbChunk { pub sig: String, pub pubkey: String, pub message: StacksSignerMessage, - pub received_at: u64, - pub received_at_block: BlockIdentifier, } diff --git a/components/client/typescript/src/schemas/stacks/payload.ts b/components/client/typescript/src/schemas/stacks/payload.ts index ef5969eb3..5d69d59a5 100644 --- a/components/client/typescript/src/schemas/stacks/payload.ts +++ b/components/client/typescript/src/schemas/stacks/payload.ts @@ -102,7 +102,11 @@ export const StacksEventSchema = Type.Object({ }); export type StacksEvent = Static; -export const StacksNonConsensusEventSchema = Type.Union([StacksSignerMessageEventSchema]); +export const StacksNonConsensusEventSchema = Type.Object({ + payload: Type.Union([StacksSignerMessageEventSchema]), + received_at: Type.Integer(), + received_at_block: BlockIdentifierSchema, +}); export type StacksNonConsensusEvent = Static; export const StacksPayloadSchema = Type.Object({ diff --git a/components/client/typescript/src/schemas/stacks/signers.ts b/components/client/typescript/src/schemas/stacks/signers.ts index 701486901..2982f7044 100644 --- a/components/client/typescript/src/schemas/stacks/signers.ts +++ b/components/client/typescript/src/schemas/stacks/signers.ts @@ -1,6 +1,4 @@ import { Static, Type } from '@fastify/type-provider-typebox'; -import { BlockIdentifierSchema } from '../common'; -import { StacksTransactionSchema } from './payload'; export const StacksNakamotoBlockHeaderSchema = Type.Object({ version: Type.Integer(), @@ -19,7 +17,8 @@ export type StacksNakamotoBlockHeader = Static; @@ -94,11 +93,12 @@ export const StacksSignerMessageSchema = Type.Union([ export type StacksSignerMessage = Static; export const StacksSignerMessageEventSchema = Type.Object({ - contract: Type.String(), - sig: Type.String(), - pubkey: Type.String(), - message: StacksSignerMessageSchema, - received_at: Type.Integer(), - received_at_block: BlockIdentifierSchema, + type: Type.Literal('signer_message'), + data: Type.Object({ + contract: Type.String(), + sig: Type.String(), + pubkey: Type.String(), + message: StacksSignerMessageSchema, + }), }); export type StacksSignerMessageEvent = Static; From bddbb730074d5390b3efaa7498725de82a3ba7d2 Mon Sep 17 00:00:00 2001 From: Rafael Cardenas Date: Wed, 23 Oct 2024 11:30:20 -0600 Subject: [PATCH 08/15] fix: 0x prefix --- .../chainhook-sdk/src/indexer/stacks/mod.rs | 40 +++++++++++-------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/components/chainhook-sdk/src/indexer/stacks/mod.rs b/components/chainhook-sdk/src/indexer/stacks/mod.rs index 1c9308dc8..258be7c5b 100644 --- a/components/chainhook-sdk/src/indexer/stacks/mod.rs +++ b/components/chainhook-sdk/src/indexer/stacks/mod.rs @@ -688,8 +688,8 @@ pub fn standardize_stacks_stackerdb_chunks( SignerMessage::BlockResponse(block_response) => match block_response { BlockResponse::Accepted((block_hash, sig)) => StacksSignerMessage::BlockResponse( BlockResponseData::Accepted(BlockAcceptedResponse { - signer_signature_hash: block_hash.to_hex(), - sig: sig.to_hex(), + signer_signature_hash: format!("0x{}", block_hash.to_hex()), + sig: format!("0x{}", sig.to_hex()), }), ), BlockResponse::Rejected(block_rejection) => StacksSignerMessage::BlockResponse( @@ -735,9 +735,9 @@ pub fn standardize_stacks_stackerdb_chunks( } RejectCode::TestingDirective => BlockRejectReasonCode::TestingDirective, }, - signer_signature_hash: block_rejection.signer_signature_hash.to_hex(), + signer_signature_hash: format!("0x{}", block_rejection.signer_signature_hash.to_hex()), chain_id: block_rejection.chain_id, - signature: block_rejection.signature.to_hex(), + signature: format!("0x{}", block_rejection.signature.to_hex()), }), ), }, @@ -754,8 +754,11 @@ pub fn standardize_stacks_stackerdb_chunks( }; parsed_chunks.push(StacksStackerDbChunk { contract: contract_id.clone(), - sig: slot.sig.clone(), - pubkey: get_signer_pubkey_from_stackerdb_chunk_slot(slot, &data_bytes)?, + sig: format!("0x{}", slot.sig), + pubkey: format!( + "0x{}", + get_signer_pubkey_from_stackerdb_chunk_slot(slot, &data_bytes)? + ), message, }); } @@ -775,23 +778,26 @@ pub fn standardize_stacks_nakamoto_block( version: block.header.version, chain_length: block.header.chain_length, burn_spent: block.header.burn_spent, - consensus_hash: block.header.consensus_hash.to_hex(), - parent_block_id: block.header.parent_block_id.to_hex(), - tx_merkle_root: block.header.tx_merkle_root.to_hex(), - state_index_root: block.header.state_index_root.to_hex(), + consensus_hash: format!("0x{}", block.header.consensus_hash.to_hex()), + parent_block_id: format!("0x{}", block.header.parent_block_id.to_hex()), + tx_merkle_root: format!("0x{}", block.header.tx_merkle_root.to_hex()), + state_index_root: format!("0x{}", block.header.state_index_root.to_hex()), timestamp: block.header.timestamp, - miner_signature: block.header.miner_signature.to_hex(), + miner_signature: format!("0x{}", block.header.miner_signature.to_hex()), signer_signature: block .header .signer_signature .iter() - .map(|s| s.to_hex()) + .map(|s| format!("0x{}", s.to_hex())) .collect(), - pox_treatment: block - .header - .pox_treatment - .serialize_to_vec() - .to_hex_string(Case::Lower), + pox_treatment: format!( + "0x{}", + block + .header + .pox_treatment + .serialize_to_vec() + .to_hex_string(Case::Lower) + ), }, // TODO(rafaelcr): Parse and return transactions. transactions: vec![], From 193ddd57f1f130a667d7689057296467654c89a5 Mon Sep 17 00:00:00 2001 From: Rafael Cardenas Date: Wed, 23 Oct 2024 11:40:28 -0600 Subject: [PATCH 09/15] fix: payload serialization --- components/chainhook-sdk/src/chainhooks/stacks/mod.rs | 4 +++- components/chainhook-types-rs/src/signers.rs | 2 +- components/client/typescript/src/schemas/stacks/signers.ts | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/components/chainhook-sdk/src/chainhooks/stacks/mod.rs b/components/chainhook-sdk/src/chainhooks/stacks/mod.rs index d1cf96ab2..1d1d53162 100644 --- a/components/chainhook-sdk/src/chainhooks/stacks/mod.rs +++ b/components/chainhook-sdk/src/chainhooks/stacks/mod.rs @@ -1102,7 +1102,9 @@ fn serialize_stacks_non_consensus_event( use chainhook_types::StacksNonConsensusEventPayloadData; let payload = match &event.payload { - StacksNonConsensusEventPayloadData::SignerMessage(chunk) => chunk, + StacksNonConsensusEventPayloadData::SignerMessage(chunk) => { + json!({"type": "SignerMessage", "data": chunk}) + } }; json!({ "payload": payload, diff --git a/components/chainhook-types-rs/src/signers.rs b/components/chainhook-types-rs/src/signers.rs index 3fcd15cb3..e2b52ecbb 100644 --- a/components/chainhook-types-rs/src/signers.rs +++ b/components/chainhook-types-rs/src/signers.rs @@ -1,4 +1,4 @@ -use crate::{BlockIdentifier, StacksTransactionData}; +use crate::StacksTransactionData; #[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] pub struct NakamotoBlockHeaderData { diff --git a/components/client/typescript/src/schemas/stacks/signers.ts b/components/client/typescript/src/schemas/stacks/signers.ts index 2982f7044..2a094424b 100644 --- a/components/client/typescript/src/schemas/stacks/signers.ts +++ b/components/client/typescript/src/schemas/stacks/signers.ts @@ -93,7 +93,7 @@ export const StacksSignerMessageSchema = Type.Union([ export type StacksSignerMessage = Static; export const StacksSignerMessageEventSchema = Type.Object({ - type: Type.Literal('signer_message'), + type: Type.Literal('SignerMessage'), data: Type.Object({ contract: Type.String(), sig: Type.String(), From 08f53b26aa8ed9adc79a37f4e22c53690ae3628f Mon Sep 17 00:00:00 2001 From: Rafael Cardenas Date: Wed, 23 Oct 2024 11:40:54 -0600 Subject: [PATCH 10/15] chore: bump client ver --- components/client/typescript/package-lock.json | 4 ++-- components/client/typescript/package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/components/client/typescript/package-lock.json b/components/client/typescript/package-lock.json index 8011fe4b0..9be7a4d6c 100644 --- a/components/client/typescript/package-lock.json +++ b/components/client/typescript/package-lock.json @@ -1,12 +1,12 @@ { "name": "@hirosystems/chainhook-client", - "version": "2.0.0", + "version": "2.1.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@hirosystems/chainhook-client", - "version": "2.0.0", + "version": "2.1.0", "license": "Apache 2.0", "dependencies": { "@fastify/type-provider-typebox": "^3.2.0", diff --git a/components/client/typescript/package.json b/components/client/typescript/package.json index f93033738..28b6906b2 100644 --- a/components/client/typescript/package.json +++ b/components/client/typescript/package.json @@ -1,6 +1,6 @@ { "name": "@hirosystems/chainhook-client", - "version": "2.0.0", + "version": "2.1.0", "description": "Chainhook TypeScript client", "main": "./dist/index.js", "typings": "./dist/index.d.ts", From d45af4d10582af0c904beaa6f6908de095db172f Mon Sep 17 00:00:00 2001 From: Rafael Cardenas Date: Wed, 23 Oct 2024 11:53:43 -0600 Subject: [PATCH 11/15] fix: tests --- components/chainhook-sdk/src/chainhooks/stacks/mod.rs | 1 - .../tests/fixtures/stacks/testnet/occurrence.json | 3 ++- components/chainhook-sdk/src/indexer/stacks/tests.rs | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/components/chainhook-sdk/src/chainhooks/stacks/mod.rs b/components/chainhook-sdk/src/chainhooks/stacks/mod.rs index 1d1d53162..a59ec8487 100644 --- a/components/chainhook-sdk/src/chainhooks/stacks/mod.rs +++ b/components/chainhook-sdk/src/chainhooks/stacks/mod.rs @@ -1094,7 +1094,6 @@ pub fn evaluate_stacks_predicate_on_transaction<'a>( } } -#[cfg(feature = "stacks-signers")] fn serialize_stacks_non_consensus_event( event: &StacksNonConsensusEventData, _ctx: &Context, diff --git a/components/chainhook-sdk/src/chainhooks/tests/fixtures/stacks/testnet/occurrence.json b/components/chainhook-sdk/src/chainhooks/tests/fixtures/stacks/testnet/occurrence.json index 9fb468e6a..55ae6171b 100644 --- a/components/chainhook-sdk/src/chainhooks/tests/fixtures/stacks/testnet/occurrence.json +++ b/components/chainhook-sdk/src/chainhooks/tests/fixtures/stacks/testnet/occurrence.json @@ -1211,5 +1211,6 @@ } ] } - ] + ], + "events": [] } \ No newline at end of file diff --git a/components/chainhook-sdk/src/indexer/stacks/tests.rs b/components/chainhook-sdk/src/indexer/stacks/tests.rs index b26a284be..e5ae1bf98 100644 --- a/components/chainhook-sdk/src/indexer/stacks/tests.rs +++ b/components/chainhook-sdk/src/indexer/stacks/tests.rs @@ -420,17 +420,17 @@ fn stackerdb_chunks_covert_into_signer_messages() { assert_eq!(message.contract, "signers-1-1"); assert_eq!( message.pubkey, - "03c76290f48909b4d49e111d69236a138ce96df3e05f709e425153d99f4fe671b4" + "0x03c76290f48909b4d49e111d69236a138ce96df3e05f709e425153d99f4fe671b4" ); - assert_eq!(message.sig, "01fc3c06f6e0ae5b13c9bb53763661817e55c8e7f1ecab8b4d4b65b283d2dd39f0099e3ea1e25e765f4f0e1dfb0a432309a16a2ec10940e1a14cb9e9b1cbf27edc"); + assert_eq!(message.sig, "0x01fc3c06f6e0ae5b13c9bb53763661817e55c8e7f1ecab8b4d4b65b283d2dd39f0099e3ea1e25e765f4f0e1dfb0a432309a16a2ec10940e1a14cb9e9b1cbf27edc"); match &message.message { StacksSignerMessage::BlockResponse(block_response_data) => match block_response_data { BlockResponseData::Accepted(block_accepted_response) => { - assert_eq!(block_accepted_response.sig, "005eb55250597b25acbf99d3dd3c2fa8189046e1b5d21309a44cbaf2b327c09b0159a01ed3f0094bfa9e5f72f5d894e12ce252081eab5396eb8bba137bddfc365b"); + assert_eq!(block_accepted_response.sig, "0x005eb55250597b25acbf99d3dd3c2fa8189046e1b5d21309a44cbaf2b327c09b0159a01ed3f0094bfa9e5f72f5d894e12ce252081eab5396eb8bba137bddfc365b"); assert_eq!( block_accepted_response.signer_signature_hash, - "74aff146904763a787aa14c614d0dd1fc63b537bdb2fd351cdf881f6db75f986" + "0x74aff146904763a787aa14c614d0dd1fc63b537bdb2fd351cdf881f6db75f986" ); } _ => assert!(false), From f77bfb6d593e20175d4bf66afa98cca2bf3472f1 Mon Sep 17 00:00:00 2001 From: Rafael Cardenas Date: Wed, 23 Oct 2024 12:56:24 -0600 Subject: [PATCH 12/15] fix: index block hash --- .../chainhook-sdk/src/indexer/stacks/mod.rs | 54 ++++++++++++++++--- components/chainhook-types-rs/src/signers.rs | 3 +- .../typescript/src/schemas/stacks/signers.ts | 2 + 3 files changed, 51 insertions(+), 8 deletions(-) diff --git a/components/chainhook-sdk/src/indexer/stacks/mod.rs b/components/chainhook-sdk/src/indexer/stacks/mod.rs index 258be7c5b..281aa9f12 100644 --- a/components/chainhook-sdk/src/indexer/stacks/mod.rs +++ b/components/chainhook-sdk/src/indexer/stacks/mod.rs @@ -680,7 +680,7 @@ pub fn standardize_stacks_stackerdb_chunks( let message = match signer_message { SignerMessage::BlockProposal(block_proposal) => { StacksSignerMessage::BlockProposal(BlockProposalData { - block: standardize_stacks_nakamoto_block(&block_proposal.block), + block: standardize_stacks_nakamoto_block(&block_proposal.block)?, burn_height: block_proposal.burn_height, reward_cycle: block_proposal.reward_cycle, }) @@ -735,7 +735,10 @@ pub fn standardize_stacks_stackerdb_chunks( } RejectCode::TestingDirective => BlockRejectReasonCode::TestingDirective, }, - signer_signature_hash: format!("0x{}", block_rejection.signer_signature_hash.to_hex()), + signer_signature_hash: format!( + "0x{}", + block_rejection.signer_signature_hash.to_hex() + ), chain_id: block_rejection.chain_id, signature: format!("0x{}", block_rejection.signature.to_hex()), }), @@ -743,7 +746,7 @@ pub fn standardize_stacks_stackerdb_chunks( }, SignerMessage::BlockPushed(nakamoto_block) => { StacksSignerMessage::BlockPushed(BlockPushedData { - block: standardize_stacks_nakamoto_block(&nakamoto_block), + block: standardize_stacks_nakamoto_block(&nakamoto_block)?, }) } SignerMessage::MockSignature(_) @@ -769,11 +772,12 @@ pub fn standardize_stacks_stackerdb_chunks( #[cfg(feature = "stacks-signers")] pub fn standardize_stacks_nakamoto_block( block: &stacks_codec::codec::NakamotoBlock, -) -> NakamotoBlockData { +) -> Result { use miniscript::bitcoin::hex::Case; use miniscript::bitcoin::hex::DisplayHex; - NakamotoBlockData { + let block_hash = get_nakamoto_block_hash(block)?; + Ok(NakamotoBlockData { header: NakamotoBlockHeaderData { version: block.header.version, chain_length: block.header.chain_length, @@ -799,9 +803,45 @@ pub fn standardize_stacks_nakamoto_block( .to_hex_string(Case::Lower) ), }, + block_hash: block_hash.clone(), + index_block_hash: get_nakamoto_index_block_hash(&block_hash, &block.header.consensus_hash)?, // TODO(rafaelcr): Parse and return transactions. transactions: vec![], - } + }) +} + +#[cfg(feature = "stacks-signers")] +fn get_nakamoto_block_hash(block: &stacks_codec::codec::NakamotoBlock) -> Result { + use clarity::util::hash::Sha512Trunc256Sum; + + let mut block_header_bytes = vec![block.header.version]; + block_header_bytes.extend(block.header.chain_length.to_be_bytes()); + block_header_bytes.extend(block.header.burn_spent.to_be_bytes()); + block_header_bytes.extend(block.header.consensus_hash.as_bytes()); + block_header_bytes.extend(block.header.parent_block_id.as_bytes()); + block_header_bytes.extend(block.header.tx_merkle_root.as_bytes()); + block_header_bytes.extend(block.header.state_index_root.as_bytes()); + block_header_bytes.extend(block.header.timestamp.to_be_bytes()); + block_header_bytes.extend(block.header.miner_signature.as_bytes()); + block_header_bytes.extend(block.header.pox_treatment.serialize_to_vec()); + + let hash = Sha512Trunc256Sum::from_data(&block_header_bytes).to_bytes(); + Ok(format!("0x{}", hex::encode(hash))) +} + +#[cfg(feature = "stacks-signers")] +fn get_nakamoto_index_block_hash( + block_hash: &String, + consensus_hash: &clarity::types::chainstate::ConsensusHash, +) -> Result { + use clarity::util::hash::Sha512Trunc256Sum; + + let mut bytes = hex::decode(block_hash[2..].to_string()) + .map_err(|e| format!("unable to decode block hash: {e}"))?; + bytes.extend(consensus_hash.as_bytes()); + + let hash = Sha512Trunc256Sum::from_data(&bytes).to_bytes(); + Ok(format!("0x{}", hex::encode(hash))) } #[cfg(feature = "stacks-signers")] @@ -819,7 +859,7 @@ pub fn get_signer_pubkey_from_stackerdb_chunk_slot( }; let mut digest_bytes = slot.slot_id.to_be_bytes().to_vec(); - digest_bytes.extend(slot.slot_version.to_be_bytes().to_vec()); + digest_bytes.extend(slot.slot_version.to_be_bytes()); let data_bytes_hashed = Sha512Trunc256Sum::from_data(&data_bytes).to_bytes(); digest_bytes.extend(data_bytes_hashed); let digest = Sha512Trunc256Sum::from_data(&digest_bytes).to_bytes(); diff --git a/components/chainhook-types-rs/src/signers.rs b/components/chainhook-types-rs/src/signers.rs index e2b52ecbb..131a32b67 100644 --- a/components/chainhook-types-rs/src/signers.rs +++ b/components/chainhook-types-rs/src/signers.rs @@ -18,12 +18,13 @@ pub struct NakamotoBlockHeaderData { #[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] pub struct NakamotoBlockData { pub header: NakamotoBlockHeaderData, + pub block_hash: String, + pub index_block_hash: String, pub transactions: Vec, } #[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] pub struct BlockProposalData { - // TODO(rafaelcr): Include `block_hash` and `index_block_hash`. pub block: NakamotoBlockData, pub burn_height: u64, pub reward_cycle: u64, diff --git a/components/client/typescript/src/schemas/stacks/signers.ts b/components/client/typescript/src/schemas/stacks/signers.ts index 2a094424b..caa0e2e3a 100644 --- a/components/client/typescript/src/schemas/stacks/signers.ts +++ b/components/client/typescript/src/schemas/stacks/signers.ts @@ -17,6 +17,8 @@ export type StacksNakamotoBlockHeader = Static Date: Wed, 23 Oct 2024 13:01:11 -0600 Subject: [PATCH 13/15] fix: timestamp at ms --- components/chainhook-sdk/src/chainhooks/stacks/mod.rs | 4 ++-- components/chainhook-sdk/src/indexer/mod.rs | 4 ++-- components/chainhook-sdk/src/observer/http.rs | 2 +- components/chainhook-types-rs/src/rosetta.rs | 2 +- components/client/typescript/src/schemas/stacks/payload.ts | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/components/chainhook-sdk/src/chainhooks/stacks/mod.rs b/components/chainhook-sdk/src/chainhooks/stacks/mod.rs index a59ec8487..1eb2c7fbf 100644 --- a/components/chainhook-sdk/src/chainhooks/stacks/mod.rs +++ b/components/chainhook-sdk/src/chainhooks/stacks/mod.rs @@ -859,7 +859,7 @@ pub fn evaluate_stacks_predicate_on_non_consensus_events<'a>( StacksPredicate::SignerMessage(StacksSignerMessagePredicate::AfterTimestamp( timestamp, )) => { - if event.received_at >= *timestamp { + if event.received_at_ms >= *timestamp { occurrences.push(event); } } @@ -1107,7 +1107,7 @@ fn serialize_stacks_non_consensus_event( }; json!({ "payload": payload, - "received_at": event.received_at, + "received_at": event.received_at_ms, "received_at_block": event.received_at_block, }) } diff --git a/components/chainhook-sdk/src/indexer/mod.rs b/components/chainhook-sdk/src/indexer/mod.rs index 9aaea364f..cc257dca5 100644 --- a/components/chainhook-sdk/src/indexer/mod.rs +++ b/components/chainhook-sdk/src/indexer/mod.rs @@ -171,7 +171,7 @@ impl Indexer { pub fn handle_stacks_marshalled_stackerdb_chunk( &mut self, marshalled_stackerdb_chunks: JsonValue, - receipt_time: u64, + receipt_time_ms: u128, ctx: &Context, ) -> Result, String> { use chainhook_types::{ @@ -192,7 +192,7 @@ impl Indexer { .into_iter() .map(|chunk| StacksNonConsensusEventData { payload: StacksNonConsensusEventPayloadData::SignerMessage(chunk), - received_at: receipt_time, + received_at_ms: receipt_time_ms as u64, received_at_block: chain_tip.clone(), }) .collect(), diff --git a/components/chainhook-sdk/src/observer/http.rs b/components/chainhook-sdk/src/observer/http.rs index 154eac688..5a5938f51 100644 --- a/components/chainhook-sdk/src/observer/http.rs +++ b/components/chainhook-sdk/src/observer/http.rs @@ -196,7 +196,7 @@ pub fn handle_stackerdb_chunks( }; let chain_event = match indexer_rw_lock.inner().write() { Ok(mut indexer) => indexer - .handle_stacks_marshalled_stackerdb_chunk(payload.into_inner(), epoch.as_secs(), ctx), + .handle_stacks_marshalled_stackerdb_chunk(payload.into_inner(), epoch.as_millis(), ctx), Err(e) => { return error_response(format!("Unable to acquire background_job_tx: {e}"), ctx); } diff --git a/components/chainhook-types-rs/src/rosetta.rs b/components/chainhook-types-rs/src/rosetta.rs index 8507a70c6..9acb4c250 100644 --- a/components/chainhook-types-rs/src/rosetta.rs +++ b/components/chainhook-types-rs/src/rosetta.rs @@ -673,7 +673,7 @@ pub enum StacksNonConsensusEventPayloadData { #[derive(Clone, Debug, PartialEq, Serialize)] pub struct StacksNonConsensusEventData { pub payload: StacksNonConsensusEventPayloadData, - pub received_at: u64, + pub received_at_ms: u64, pub received_at_block: BlockIdentifier, } diff --git a/components/client/typescript/src/schemas/stacks/payload.ts b/components/client/typescript/src/schemas/stacks/payload.ts index 5d69d59a5..3bf1a980a 100644 --- a/components/client/typescript/src/schemas/stacks/payload.ts +++ b/components/client/typescript/src/schemas/stacks/payload.ts @@ -104,7 +104,7 @@ export type StacksEvent = Static; export const StacksNonConsensusEventSchema = Type.Object({ payload: Type.Union([StacksSignerMessageEventSchema]), - received_at: Type.Integer(), + received_at_ms: Type.Integer(), received_at_block: BlockIdentifierSchema, }); export type StacksNonConsensusEvent = Static; From 176e114520e4da0254fc79c208532caebe577eec Mon Sep 17 00:00:00 2001 From: Rafael Cardenas Date: Wed, 23 Oct 2024 13:01:25 -0600 Subject: [PATCH 14/15] fix: version --- components/client/typescript/package-lock.json | 4 ++-- components/client/typescript/package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/components/client/typescript/package-lock.json b/components/client/typescript/package-lock.json index 9be7a4d6c..55291fdc9 100644 --- a/components/client/typescript/package-lock.json +++ b/components/client/typescript/package-lock.json @@ -1,12 +1,12 @@ { "name": "@hirosystems/chainhook-client", - "version": "2.1.0", + "version": "2.1.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@hirosystems/chainhook-client", - "version": "2.1.0", + "version": "2.1.1", "license": "Apache 2.0", "dependencies": { "@fastify/type-provider-typebox": "^3.2.0", diff --git a/components/client/typescript/package.json b/components/client/typescript/package.json index 28b6906b2..5cb41df87 100644 --- a/components/client/typescript/package.json +++ b/components/client/typescript/package.json @@ -1,6 +1,6 @@ { "name": "@hirosystems/chainhook-client", - "version": "2.1.0", + "version": "2.1.1", "description": "Chainhook TypeScript client", "main": "./dist/index.js", "typings": "./dist/index.d.ts", From 85093ec655c8f3b260fec983ddba14580d123766 Mon Sep 17 00:00:00 2001 From: Rafael Cardenas Date: Wed, 23 Oct 2024 13:03:19 -0600 Subject: [PATCH 15/15] fix: openapi --- docs/chainhook-openapi.json | 42 +++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/docs/chainhook-openapi.json b/docs/chainhook-openapi.json index 989f3c230..c8f04fdcd 100644 --- a/docs/chainhook-openapi.json +++ b/docs/chainhook-openapi.json @@ -1207,6 +1207,48 @@ ] } } + }, + { + "type": "object", + "oneOf": [ + { + "type": "object", + "required": [ + "after_timestamp" + ], + "properties": { + "after_timestamp": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "from_signer_pub_key" + ], + "properties": { + "from_signer_pub_key": { + "type": "string" + } + }, + "additionalProperties": false + } + ], + "required": [ + "scope" + ], + "properties": { + "scope": { + "type": "string", + "enum": [ + "signer_message" + ] + } + } } ] },