From 6627606270fdf4a0b20111eaf528992fda172e75 Mon Sep 17 00:00:00 2001 From: Matthew Little Date: Tue, 22 Oct 2024 15:18:48 +0200 Subject: [PATCH 1/2] feat: add various new nakamoto block fields to `/new_block` ingestion and `StacksPayload` --- .../service/tests/helpers/mock_stacks_node.rs | 22 +++++++- .../chainhook-sdk/src/indexer/stacks/mod.rs | 53 +++++++++++++++++++ .../indexer/tests/helpers/stacks_blocks.rs | 22 +++++++- components/chainhook-types-js/src/index.ts | 17 ++++++ components/chainhook-types-rs/src/rosetta.rs | 23 ++++++++ .../typescript/src/schemas/stacks/payload.ts | 23 ++++++++ 6 files changed, 158 insertions(+), 2 deletions(-) diff --git a/components/chainhook-cli/src/service/tests/helpers/mock_stacks_node.rs b/components/chainhook-cli/src/service/tests/helpers/mock_stacks_node.rs index a13198e36..7cc7ff22f 100644 --- a/components/chainhook-cli/src/service/tests/helpers/mock_stacks_node.rs +++ b/components/chainhook-cli/src/service/tests/helpers/mock_stacks_node.rs @@ -1,7 +1,7 @@ use crate::scan::stacks::{Record, RecordKind}; use crate::service::tests::helpers::mock_bitcoin_rpc::TipData; use chainhook_sdk::indexer::bitcoin::NewBitcoinBlock; -use chainhook_sdk::indexer::stacks::{NewBlock, NewEvent, NewTransaction}; +use chainhook_sdk::indexer::stacks::{NewBlock, NewEvent, NewTransaction, RewardSet, RewardSetSigner}; use chainhook_sdk::types::{ FTBurnEventData, FTMintEventData, FTTransferEventData, NFTBurnEventData, NFTMintEventData, NFTTransferEventData, STXBurnEventData, STXLockEventData, STXMintEventData, @@ -260,6 +260,26 @@ pub fn create_stacks_new_block( transactions: (0..4).map(create_stacks_new_transaction).collect(), events, matured_miner_rewards: vec![], + block_time: Some(12345), + signer_bitvec: Some("000800000001ff".to_owned()), + signer_signature: Some(vec!["1234".to_owned(), "2345".to_owned()]), + cycle_number: Some(1), + reward_set: Some(RewardSet { + pox_ustx_threshold: "50000".to_owned(), + rewarded_addresses: vec![], + signers: Some(vec![ + RewardSetSigner { + signing_key: "0123".to_owned(), + weight: 123, + stacked_amt: "555555".to_owned(), + }, + RewardSetSigner { + signing_key: "2345".to_owned(), + weight: 234, + stacked_amt: "6677777".to_owned(), + }, + ]), + }), } } diff --git a/components/chainhook-sdk/src/indexer/stacks/mod.rs b/components/chainhook-sdk/src/indexer/stacks/mod.rs index 25b80c7d1..005cd10c0 100644 --- a/components/chainhook-sdk/src/indexer/stacks/mod.rs +++ b/components/chainhook-sdk/src/indexer/stacks/mod.rs @@ -35,6 +35,36 @@ pub struct NewBlock { pub transactions: Vec, pub events: Vec, pub matured_miner_rewards: Vec, + + #[serde(skip_serializing_if = "Option::is_none")] + pub block_time: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub signer_bitvec: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub signer_signature: Option>, + + #[serde(skip_serializing_if = "Option::is_none")] + pub cycle_number: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub reward_set: Option, +} + +#[derive(Deserialize, Serialize)] +pub struct RewardSet { + pub pox_ustx_threshold: String, + pub rewarded_addresses: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub signers: Option>, +} + +#[derive(Deserialize, Serialize)] +pub struct RewardSetSigner { + pub signing_key: String, + pub weight: u32, + pub stacked_amt: String, } #[derive(Deserialize, Serialize, Default, Clone)] @@ -432,6 +462,29 @@ pub fn standardize_stacks_block( pox_cycle_length: pox_cycle_length.try_into().unwrap(), confirm_microblock_identifier, stacks_block_hash: block.block_hash.clone(), + + block_time: block.block_time, + // TODO: decode `signer_bitvec` into an easy to use bit string representation (e.g. "01010101") + signer_bitvec: block.signer_bitvec.clone(), + signer_signature: block.signer_signature.clone(), + + cycle_number: block.cycle_number, + reward_set: block.reward_set.as_ref().and_then(|r| { + Some(StacksBlockMetadataRewardSet { + pox_ustx_threshold: r.pox_ustx_threshold.clone(), + rewarded_addresses: r.rewarded_addresses.clone(), + signers: r.signers.as_ref().map(|signers| { + signers + .into_iter() + .map(|signer| StacksBlockMetadataRewardSetSigner { + signing_key: signer.signing_key.clone(), + weight: signer.weight, + stacked_amt: signer.stacked_amt.clone(), + }) + .collect() + }), + }) + }), }, transactions, }; diff --git a/components/chainhook-sdk/src/indexer/tests/helpers/stacks_blocks.rs b/components/chainhook-sdk/src/indexer/tests/helpers/stacks_blocks.rs index 95e4208a4..d41d9cb47 100644 --- a/components/chainhook-sdk/src/indexer/tests/helpers/stacks_blocks.rs +++ b/components/chainhook-sdk/src/indexer/tests/helpers/stacks_blocks.rs @@ -1,6 +1,6 @@ use super::BlockEvent; use chainhook_types::{ - BlockIdentifier, StacksBlockData, StacksBlockMetadata, StacksTransactionData, + BlockIdentifier, StacksBlockData, StacksBlockMetadata, StacksBlockMetadataRewardSet, StacksBlockMetadataRewardSetSigner, StacksTransactionData }; pub fn generate_test_stacks_block( @@ -72,6 +72,26 @@ pub fn generate_test_stacks_block( pox_cycle_length: 100, confirm_microblock_identifier, stacks_block_hash: String::new(), + block_time: Some(12345), + signer_bitvec: Some("1010101010101".to_owned()), + signer_signature: Some(vec!["1234".to_owned(), "2345".to_owned()]), + cycle_number: Some(1), + reward_set: Some(StacksBlockMetadataRewardSet { + pox_ustx_threshold: "50000".to_owned(), + rewarded_addresses: vec![], + signers: Some(vec![ + StacksBlockMetadataRewardSetSigner { + signing_key: "0123".to_owned(), + weight: 123, + stacked_amt: "555555".to_owned(), + }, + StacksBlockMetadataRewardSetSigner { + signing_key: "2345".to_owned(), + weight: 234, + stacked_amt: "6677777".to_owned(), + }, + ]), + }), }, }) } diff --git a/components/chainhook-types-js/src/index.ts b/components/chainhook-types-js/src/index.ts index 2075a4138..dc4473582 100644 --- a/components/chainhook-types-js/src/index.ts +++ b/components/chainhook-types-js/src/index.ts @@ -694,6 +694,23 @@ export interface StacksBlockMetadata { * @memberof StacksBlockMetadata */ pox_cycle_length: number; + + block_time?: number | null; + signer_bitvec?: string | null; + signer_signature?: string[] | null; + cycle_number?: number | null; + reward_set?: { + pox_ustx_threshold: string; + rewarded_addresses: string[]; + signers?: { + signing_key: string; + weight: number; + stacked_amt: string; + }[] | null; + start_cycle_state: { + missed_reward_slots: []; + }; + } | null; } /** diff --git a/components/chainhook-types-rs/src/rosetta.rs b/components/chainhook-types-rs/src/rosetta.rs index 9ebbb18e6..6e3d6750c 100644 --- a/components/chainhook-types-rs/src/rosetta.rs +++ b/components/chainhook-types-rs/src/rosetta.rs @@ -114,6 +114,29 @@ pub struct StacksBlockMetadata { pub pox_cycle_length: u32, pub confirm_microblock_identifier: Option, pub stacks_block_hash: String, + + // Fields included in Nakamoto block headers + pub block_time: Option, + pub signer_bitvec: Option, + pub signer_signature: Option>, + + // Available starting in epoch3, only included in blocks where the pox cycle rewards are first calculated + pub cycle_number: Option, + pub reward_set: Option, +} + +#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] +pub struct StacksBlockMetadataRewardSet { + pub pox_ustx_threshold: String, + pub rewarded_addresses: Vec, + pub signers: Option>, +} + +#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] +pub struct StacksBlockMetadataRewardSetSigner { + pub signing_key: String, + pub weight: u32, + pub stacked_amt: String, } /// BitcoinBlock contain an array of Transactions that occurred at a particular diff --git a/components/client/typescript/src/schemas/stacks/payload.ts b/components/client/typescript/src/schemas/stacks/payload.ts index db1c6ea8a..cc1653e1e 100644 --- a/components/client/typescript/src/schemas/stacks/payload.ts +++ b/components/client/typescript/src/schemas/stacks/payload.ts @@ -66,6 +66,29 @@ export const StacksEventMetadataSchema = Type.Object({ pox_cycle_length: Type.Integer(), pox_cycle_position: Type.Integer(), stacks_block_hash: Type.String(), + + // Fields included in Nakamoto block headers + block_time: Nullable(Type.Integer()), + signer_bitvec: Nullable(Type.String()), + signer_signature: Nullable(Type.Array(Type.String())), + + // Available starting in epoch3, only included in blocks where the pox cycle rewards are first calculated + cycle_number: Nullable(Type.Integer()), + reward_set: Nullable( + Type.Object({ + pox_ustx_threshold: Type.String(), + rewarded_addresses: Type.Array(Type.String()), + signers: Nullable( + Type.Array( + Type.Object({ + signing_key: Type.String(), + weight: Type.Integer(), + stacked_amt: Type.String(), + }) + ) + ), + }) + ), }); export type StacksEventMetadata = Static; From 212a8e41649a6105b88bc29bd9941c2317f991e0 Mon Sep 17 00:00:00 2001 From: Matthew Little Date: Tue, 22 Oct 2024 16:46:23 +0200 Subject: [PATCH 2/2] chore: fix tests --- Cargo.lock | 11 +++ components/chainhook-sdk/Cargo.toml | 1 + .../fixtures/stacks/testnet/occurrence.json | 98 ++++++++++++++++--- .../chainhook-sdk/src/chainhooks/tests/mod.rs | 9 +- 4 files changed, 100 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 80b5f722f..95e4025eb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -124,6 +124,16 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" +[[package]] +name = "assert-json-diff" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "async-stream" version = "0.3.5" @@ -503,6 +513,7 @@ dependencies = [ name = "chainhook-sdk" version = "0.12.11" dependencies = [ + "assert-json-diff", "base58", "base64 0.21.7", "bitcoincore-rpc", diff --git a/components/chainhook-sdk/Cargo.toml b/components/chainhook-sdk/Cargo.toml index ec6b2e6a2..f7e2ca17d 100644 --- a/components/chainhook-sdk/Cargo.toml +++ b/components/chainhook-sdk/Cargo.toml @@ -47,6 +47,7 @@ prometheus = "0.13.3" chainhook-types = { path = "../chainhook-types-rs" } [dev-dependencies] +assert-json-diff = "2.0.2" test-case = "3.1.0" [features] diff --git a/components/chainhook-sdk/src/chainhooks/tests/fixtures/stacks/testnet/occurrence.json b/components/chainhook-sdk/src/chainhooks/tests/fixtures/stacks/testnet/occurrence.json index 9dc863fc9..9fb468e6a 100644 --- a/components/chainhook-sdk/src/chainhooks/tests/fixtures/stacks/testnet/occurrence.json +++ b/components/chainhook-sdk/src/chainhooks/tests/fixtures/stacks/testnet/occurrence.json @@ -14,7 +14,12 @@ "pox_cycle_index": 415, "pox_cycle_length": 1050, "pox_cycle_position": 1033, - "stacks_block_hash": "0x" + "stacks_block_hash": "0x", + "block_time": null, + "cycle_number": null, + "reward_set": null, + "signer_bitvec": null, + "signer_signature": null }, "parent_block_identifier": { "hash": "0x", @@ -96,7 +101,12 @@ "pox_cycle_index": 415, "pox_cycle_length": 1050, "pox_cycle_position": 1033, - "stacks_block_hash": "0x" + "stacks_block_hash": "0x", + "block_time": null, + "cycle_number": null, + "reward_set": null, + "signer_bitvec": null, + "signer_signature": null }, "parent_block_identifier": { "hash": "0x", @@ -177,7 +187,12 @@ "pox_cycle_index": 415, "pox_cycle_length": 1050, "pox_cycle_position": 1033, - "stacks_block_hash": "0x" + "stacks_block_hash": "0x", + "block_time": null, + "cycle_number": null, + "reward_set": null, + "signer_bitvec": null, + "signer_signature": null }, "parent_block_identifier": { "hash": "0x", @@ -259,7 +274,12 @@ "pox_cycle_index": 415, "pox_cycle_length": 1050, "pox_cycle_position": 1033, - "stacks_block_hash": "0x" + "stacks_block_hash": "0x", + "block_time": null, + "cycle_number": null, + "reward_set": null, + "signer_bitvec": null, + "signer_signature": null }, "parent_block_identifier": { "hash": "0x", @@ -340,7 +360,12 @@ "pox_cycle_index": 415, "pox_cycle_length": 1050, "pox_cycle_position": 1033, - "stacks_block_hash": "0x" + "stacks_block_hash": "0x", + "block_time": null, + "cycle_number": null, + "reward_set": null, + "signer_bitvec": null, + "signer_signature": null }, "parent_block_identifier": { "hash": "0x", @@ -423,7 +448,12 @@ "pox_cycle_index": 415, "pox_cycle_length": 1050, "pox_cycle_position": 1033, - "stacks_block_hash": "0x" + "stacks_block_hash": "0x", + "block_time": null, + "cycle_number": null, + "reward_set": null, + "signer_bitvec": null, + "signer_signature": null }, "parent_block_identifier": { "hash": "0x", @@ -505,7 +535,12 @@ "pox_cycle_index": 415, "pox_cycle_length": 1050, "pox_cycle_position": 1033, - "stacks_block_hash": "0x" + "stacks_block_hash": "0x", + "block_time": null, + "cycle_number": null, + "reward_set": null, + "signer_bitvec": null, + "signer_signature": null }, "parent_block_identifier": { "hash": "0x", @@ -587,7 +622,12 @@ "pox_cycle_index": 415, "pox_cycle_length": 1050, "pox_cycle_position": 1033, - "stacks_block_hash": "0x" + "stacks_block_hash": "0x", + "block_time": null, + "cycle_number": null, + "reward_set": null, + "signer_bitvec": null, + "signer_signature": null }, "parent_block_identifier": { "hash": "0x", @@ -670,7 +710,12 @@ "pox_cycle_index": 415, "pox_cycle_length": 1050, "pox_cycle_position": 1033, - "stacks_block_hash": "0x" + "stacks_block_hash": "0x", + "block_time": null, + "cycle_number": null, + "reward_set": null, + "signer_bitvec": null, + "signer_signature": null }, "parent_block_identifier": { "hash": "0x", @@ -752,7 +797,12 @@ "pox_cycle_index": 415, "pox_cycle_length": 1050, "pox_cycle_position": 1033, - "stacks_block_hash": "0x" + "stacks_block_hash": "0x", + "block_time": null, + "cycle_number": null, + "reward_set": null, + "signer_bitvec": null, + "signer_signature": null }, "parent_block_identifier": { "hash": "0x", @@ -834,7 +884,12 @@ "pox_cycle_index": 415, "pox_cycle_length": 1050, "pox_cycle_position": 1033, - "stacks_block_hash": "0x" + "stacks_block_hash": "0x", + "block_time": null, + "cycle_number": null, + "reward_set": null, + "signer_bitvec": null, + "signer_signature": null }, "parent_block_identifier": { "hash": "0x", @@ -916,7 +971,12 @@ "pox_cycle_index": 415, "pox_cycle_length": 1050, "pox_cycle_position": 1033, - "stacks_block_hash": "0x" + "stacks_block_hash": "0x", + "block_time": null, + "cycle_number": null, + "reward_set": null, + "signer_bitvec": null, + "signer_signature": null }, "parent_block_identifier": { "hash": "0x", @@ -998,7 +1058,12 @@ "pox_cycle_index": 415, "pox_cycle_length": 1050, "pox_cycle_position": 1033, - "stacks_block_hash": "0x" + "stacks_block_hash": "0x", + "block_time": null, + "cycle_number": null, + "reward_set": null, + "signer_bitvec": null, + "signer_signature": null }, "parent_block_identifier": { "hash": "0x", @@ -1090,7 +1155,12 @@ "pox_cycle_index": 415, "pox_cycle_length": 1050, "pox_cycle_position": 1033, - "stacks_block_hash": "0x" + "stacks_block_hash": "0x", + "block_time": null, + "cycle_number": null, + "reward_set": null, + "signer_bitvec": null, + "signer_signature": null }, "parent_block_identifier": { "hash": "0x", diff --git a/components/chainhook-sdk/src/chainhooks/tests/mod.rs b/components/chainhook-sdk/src/chainhooks/tests/mod.rs index 9c5ee7625..14b0774b4 100644 --- a/components/chainhook-sdk/src/chainhooks/tests/mod.rs +++ b/components/chainhook-sdk/src/chainhooks/tests/mod.rs @@ -24,6 +24,7 @@ use crate::{ }, utils::AbstractStacksBlock, }; +use assert_json_diff::assert_json_eq; use chainhook_types::{ StacksBlockUpdate, StacksChainEvent, StacksChainUpdatedWithBlocksData, StacksNetwork, StacksTransactionData, StacksTransactionEvent, StacksTransactionEventPayload, @@ -821,11 +822,9 @@ fn test_stacks_hook_action_file_append() { handle_stacks_hook_action(trigger, &proofs, &EventObserverConfig::default(), &ctx).unwrap(); if let StacksChainhookOccurrence::File(path, bytes) = occurrence { assert_eq!(path, "./".to_string()); - let json: JsonValue = serde_json::from_slice(&bytes).unwrap(); - let obj = json.as_object().unwrap(); - let actual = serde_json::to_string_pretty(obj).unwrap(); - let expected = get_expected_occurrence(); - assert_eq!(expected, actual); + let actual: JsonValue = serde_json::from_slice(&bytes).unwrap(); + let expected: JsonValue = serde_json::from_str(&get_expected_occurrence()).unwrap(); + assert_json_eq!(expected, actual); } else { panic!("wrong occurrence type"); }