From 4d8a7735a14a9359110d3ed3a30f994eb982a9db Mon Sep 17 00:00:00 2001 From: Lasse Alm Date: Tue, 3 Dec 2024 20:19:15 +0100 Subject: [PATCH 01/12] fix serialization --- backend-rust/docker-compose.yml | 2 +- backend-rust/src/graphql_api.rs | 4 ---- backend-rust/src/indexer.rs | 24 ++++++++++++++++++++++-- 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/backend-rust/docker-compose.yml b/backend-rust/docker-compose.yml index b3f8bc648..6827f76b3 100644 --- a/backend-rust/docker-compose.yml +++ b/backend-rust/docker-compose.yml @@ -9,4 +9,4 @@ services: - ./data:/var/lib/postgresql/data environment: POSTGRES_PASSWORD: ${PGPASSWORD} - POSTGRES_DB: ccd-scan + POSTGRES_DB: ccdscan diff --git a/backend-rust/src/graphql_api.rs b/backend-rust/src/graphql_api.rs index 8f229d91f..790630f3f 100644 --- a/backend-rust/src/graphql_api.rs +++ b/backend-rust/src/graphql_api.rs @@ -4159,7 +4159,6 @@ pub fn events_from_summary( } => { vec![Event::DataRegistered(DataRegistered { data_as_hex: hex::encode(data.as_ref()), - decoded: DecodedText::from_bytes(data.as_ref()), })] } AccountTransactionEffects::BakerConfigured { @@ -4675,7 +4674,6 @@ impl TryFrom for TransactionRejectReas impl From for TransferMemo { fn from(value: concordium_rust_sdk::types::Memo) -> Self { TransferMemo { - decoded: DecodedText::from_bytes(value.as_ref()), raw_hex: hex::encode(value.as_ref()), } } @@ -4918,7 +4916,6 @@ pub struct CredentialsUpdated { #[derive(SimpleObject, serde::Serialize, serde::Deserialize)] pub struct DataRegistered { - decoded: DecodedText, data_as_hex: String, } @@ -5005,7 +5002,6 @@ impl TryFrom for NewEncrypt #[derive(SimpleObject, serde::Serialize, serde::Deserialize)] pub struct TransferMemo { - decoded: DecodedText, raw_hex: String, } diff --git a/backend-rust/src/indexer.rs b/backend-rust/src/indexer.rs index 43c013b01..bebe37d69 100644 --- a/backend-rust/src/indexer.rs +++ b/backend-rust/src/indexer.rs @@ -32,10 +32,11 @@ use prometheus_client::{ }, registry::Registry, }; -use sqlx::PgPool; +use sqlx::{Execute, PgPool}; use tokio::{time::Instant, try_join}; use tokio_util::sync::CancellationToken; use tracing::{error, info, warn}; +use base64::{encode, DecodeError}; /// Service traversing each block of the chain, indexing it into a database. /// @@ -786,6 +787,21 @@ impl PreparedBlock { context.last_finalized_hash = block.block_last_finalized.clone(); } } + // Log each argument + println!("Heights: {:?}", heights); + println!("Hashes: {:?}", hashes); + println!("Slot Times: {:?}", slot_times); + println!("Block Times: {:?}", block_times); + println!( + "Baker IDs: {:?}", + baker_ids + .iter() + .map(|id| id.map_or("NULL".to_string(), |v| v.to_string())) + .collect::>() + ); + println!("Total Amounts: {:?}", total_amounts); + println!("Total Staked: {:?}", total_staked); + println!("Cumulative Num TXs: {:?}", cumulative_num_txss); sqlx::query!( r#"INSERT INTO blocks @@ -960,6 +976,7 @@ impl PreparedBlockItem { &self, tx: &mut sqlx::Transaction<'static, sqlx::Postgres>, ) -> anyhow::Result<()> { + let tx_idx = sqlx::query_scalar!( "INSERT INTO transactions ( index, @@ -1004,8 +1021,11 @@ impl PreparedBlockItem { self.reject ) .fetch_one(tx.as_mut()) - .await?; + .await; + let json_string = serde_json::to_string(&self.events)?; + println!("{:?}", encode(json_string)); + let tx_idx = tx_idx?; // Note that this does not include account creation. We handle that when saving // the account creation event. sqlx::query!( From 2b906fcf22dcc857b49ecdeb7da8f1fbd3041725 Mon Sep 17 00:00:00 2001 From: Lasse Alm Date: Wed, 4 Dec 2024 09:36:43 +0100 Subject: [PATCH 02/12] write tests --- backend-rust/.idea/.gitignore | 5 +++++ backend-rust/.idea/backend-rust.iml | 11 +++++++++++ .../.idea/inspectionProfiles/Project_Default.xml | 13 +++++++++++++ backend-rust/.idea/modules.xml | 8 ++++++++ backend-rust/.idea/vcs.xml | 9 +++++++++ backend-rust/src/indexer.rs | 5 +---- 6 files changed, 47 insertions(+), 4 deletions(-) create mode 100644 backend-rust/.idea/.gitignore create mode 100644 backend-rust/.idea/backend-rust.iml create mode 100644 backend-rust/.idea/inspectionProfiles/Project_Default.xml create mode 100644 backend-rust/.idea/modules.xml create mode 100644 backend-rust/.idea/vcs.xml diff --git a/backend-rust/.idea/.gitignore b/backend-rust/.idea/.gitignore new file mode 100644 index 000000000..b58b603fe --- /dev/null +++ b/backend-rust/.idea/.gitignore @@ -0,0 +1,5 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ diff --git a/backend-rust/.idea/backend-rust.iml b/backend-rust/.idea/backend-rust.iml new file mode 100644 index 000000000..cf84ae4a6 --- /dev/null +++ b/backend-rust/.idea/backend-rust.iml @@ -0,0 +1,11 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/backend-rust/.idea/inspectionProfiles/Project_Default.xml b/backend-rust/.idea/inspectionProfiles/Project_Default.xml new file mode 100644 index 000000000..b2b65097d --- /dev/null +++ b/backend-rust/.idea/inspectionProfiles/Project_Default.xml @@ -0,0 +1,13 @@ + + + + \ No newline at end of file diff --git a/backend-rust/.idea/modules.xml b/backend-rust/.idea/modules.xml new file mode 100644 index 000000000..2ab035844 --- /dev/null +++ b/backend-rust/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/backend-rust/.idea/vcs.xml b/backend-rust/.idea/vcs.xml new file mode 100644 index 000000000..4ca8ad5a0 --- /dev/null +++ b/backend-rust/.idea/vcs.xml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/backend-rust/src/indexer.rs b/backend-rust/src/indexer.rs index bebe37d69..e9d4c410d 100644 --- a/backend-rust/src/indexer.rs +++ b/backend-rust/src/indexer.rs @@ -1021,11 +1021,8 @@ impl PreparedBlockItem { self.reject ) .fetch_one(tx.as_mut()) - .await; - let json_string = serde_json::to_string(&self.events)?; - println!("{:?}", encode(json_string)); + .await?; - let tx_idx = tx_idx?; // Note that this does not include account creation. We handle that when saving // the account creation event. sqlx::query!( From 300bfb6e580e22dbe1be485146f8fb22d089808c Mon Sep 17 00:00:00 2001 From: Lasse Alm Date: Wed, 4 Dec 2024 09:39:54 +0100 Subject: [PATCH 03/12] remove idea files --- backend-rust/.idea/.gitignore | 5 ----- backend-rust/.idea/backend-rust.iml | 11 ----------- .../.idea/inspectionProfiles/Project_Default.xml | 13 ------------- backend-rust/.idea/modules.xml | 8 -------- backend-rust/.idea/vcs.xml | 9 --------- 5 files changed, 46 deletions(-) delete mode 100644 backend-rust/.idea/.gitignore delete mode 100644 backend-rust/.idea/backend-rust.iml delete mode 100644 backend-rust/.idea/inspectionProfiles/Project_Default.xml delete mode 100644 backend-rust/.idea/modules.xml delete mode 100644 backend-rust/.idea/vcs.xml diff --git a/backend-rust/.idea/.gitignore b/backend-rust/.idea/.gitignore deleted file mode 100644 index b58b603fe..000000000 --- a/backend-rust/.idea/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -# Default ignored files -/shelf/ -/workspace.xml -# Editor-based HTTP Client requests -/httpRequests/ diff --git a/backend-rust/.idea/backend-rust.iml b/backend-rust/.idea/backend-rust.iml deleted file mode 100644 index cf84ae4a6..000000000 --- a/backend-rust/.idea/backend-rust.iml +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - - - - - - \ No newline at end of file diff --git a/backend-rust/.idea/inspectionProfiles/Project_Default.xml b/backend-rust/.idea/inspectionProfiles/Project_Default.xml deleted file mode 100644 index b2b65097d..000000000 --- a/backend-rust/.idea/inspectionProfiles/Project_Default.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - \ No newline at end of file diff --git a/backend-rust/.idea/modules.xml b/backend-rust/.idea/modules.xml deleted file mode 100644 index 2ab035844..000000000 --- a/backend-rust/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/backend-rust/.idea/vcs.xml b/backend-rust/.idea/vcs.xml deleted file mode 100644 index 4ca8ad5a0..000000000 --- a/backend-rust/.idea/vcs.xml +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - - - - \ No newline at end of file From b12883f8ec1c09da0b09a51ca1691d688c04bd84 Mon Sep 17 00:00:00 2001 From: Lasse Alm Date: Wed, 4 Dec 2024 09:40:45 +0100 Subject: [PATCH 04/12] remove .idea --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index b57c37c0b..02821a0c9 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,5 @@ target/ .vscode/ -.env \ No newline at end of file +.env +**/.idea \ No newline at end of file From 8dd17b94fce225816590eb5df65f682a07602ba7 Mon Sep 17 00:00:00 2001 From: Lasse Alm Date: Wed, 4 Dec 2024 09:41:18 +0100 Subject: [PATCH 05/12] fix .idea --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 02821a0c9..7d6c684f5 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,5 @@ target/ .vscode/ .env -**/.idea \ No newline at end of file + +**/.idea From aa2b8ed88f2ee75fe50b9204f2d7425c6b9f2af8 Mon Sep 17 00:00:00 2001 From: Lasse Alm Date: Wed, 4 Dec 2024 09:43:13 +0100 Subject: [PATCH 06/12] remove prints --- backend-rust/src/indexer.rs | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/backend-rust/src/indexer.rs b/backend-rust/src/indexer.rs index e9d4c410d..384012aad 100644 --- a/backend-rust/src/indexer.rs +++ b/backend-rust/src/indexer.rs @@ -787,21 +787,6 @@ impl PreparedBlock { context.last_finalized_hash = block.block_last_finalized.clone(); } } - // Log each argument - println!("Heights: {:?}", heights); - println!("Hashes: {:?}", hashes); - println!("Slot Times: {:?}", slot_times); - println!("Block Times: {:?}", block_times); - println!( - "Baker IDs: {:?}", - baker_ids - .iter() - .map(|id| id.map_or("NULL".to_string(), |v| v.to_string())) - .collect::>() - ); - println!("Total Amounts: {:?}", total_amounts); - println!("Total Staked: {:?}", total_staked); - println!("Cumulative Num TXs: {:?}", cumulative_num_txss); sqlx::query!( r#"INSERT INTO blocks From 30edce9a709abb363e890c0911d88c64423ab929 Mon Sep 17 00:00:00 2001 From: Lasse Alm Date: Wed, 4 Dec 2024 09:43:45 +0100 Subject: [PATCH 07/12] remove newline --- backend-rust/src/indexer.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/backend-rust/src/indexer.rs b/backend-rust/src/indexer.rs index 384012aad..9126b3ab3 100644 --- a/backend-rust/src/indexer.rs +++ b/backend-rust/src/indexer.rs @@ -961,7 +961,6 @@ impl PreparedBlockItem { &self, tx: &mut sqlx::Transaction<'static, sqlx::Postgres>, ) -> anyhow::Result<()> { - let tx_idx = sqlx::query_scalar!( "INSERT INTO transactions ( index, From 0e5ac2d64342e2bce8ce12c443458b05cfc68916 Mon Sep 17 00:00:00 2001 From: Lasse Alm Date: Wed, 4 Dec 2024 09:44:27 +0100 Subject: [PATCH 08/12] fix indexing --- backend-rust/src/indexer.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/backend-rust/src/indexer.rs b/backend-rust/src/indexer.rs index 9126b3ab3..43c013b01 100644 --- a/backend-rust/src/indexer.rs +++ b/backend-rust/src/indexer.rs @@ -32,11 +32,10 @@ use prometheus_client::{ }, registry::Registry, }; -use sqlx::{Execute, PgPool}; +use sqlx::PgPool; use tokio::{time::Instant, try_join}; use tokio_util::sync::CancellationToken; use tracing::{error, info, warn}; -use base64::{encode, DecodeError}; /// Service traversing each block of the chain, indexing it into a database. /// From 3467ffac7528cfd577794d9bd79c3b1ede6c5555 Mon Sep 17 00:00:00 2001 From: Lasse Alm Date: Wed, 4 Dec 2024 11:21:30 +0100 Subject: [PATCH 09/12] make decoding on api call instead of ingestion --- backend-rust/src/graphql_api.rs | 35 ++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/backend-rust/src/graphql_api.rs b/backend-rust/src/graphql_api.rs index 790630f3f..383b07b44 100644 --- a/backend-rust/src/graphql_api.rs +++ b/backend-rust/src/graphql_api.rs @@ -44,6 +44,7 @@ use sqlx::{postgres::types::PgInterval, PgPool}; use std::{error::Error, mem, str::FromStr, sync::Arc}; use tokio::{net::TcpListener, sync::broadcast}; use tokio_util::sync::CancellationToken; +use tracing::error; use transaction_metrics::TransactionMetricsQuery; const VERSION: &str = clap::crate_version!(); @@ -4915,8 +4916,24 @@ pub struct CredentialsUpdated { } #[derive(SimpleObject, serde::Serialize, serde::Deserialize)] +#[graphql(complex)] pub struct DataRegistered { - data_as_hex: String, + data_as_hex: String +} + +#[ComplexObject] +impl DataRegistered { + async fn decoded(&self) -> ApiResult { + // Attempt to decode the hex string + let decoded_data = hex::decode(&self.data_as_hex) + .map_err(|e| { + // Log an error if the decoding fails + error!("Invalid hex encoding {:?} in a controlled environment", e); + ApiError::InternalError("Failed to decode hex data".to_string()) + })?; + + Ok(DecodedText::from_bytes(decoded_data.as_slice())) + } } #[derive(SimpleObject, serde::Serialize, serde::Deserialize)] @@ -5001,10 +5018,26 @@ impl TryFrom for NewEncrypt } #[derive(SimpleObject, serde::Serialize, serde::Deserialize)] +#[graphql(complex)] pub struct TransferMemo { raw_hex: String, } +#[ComplexObject] +impl TransferMemo { + async fn decoded(&self) -> ApiResult { + // Attempt to decode the hex string + let decoded_data = hex::decode(&self.raw_hex) + .map_err(|e| { + // Log an error if the decoding fails + error!("Invalid hex encoding {:?} in a controlled environment", e); + ApiError::InternalError("Failed to decode hex data".to_string()) + })?; + + Ok(DecodedText::from_bytes(decoded_data.as_slice())) + } +} + #[derive(SimpleObject, serde::Serialize, serde::Deserialize)] pub struct TransferredWithSchedule { from_account_address: AccountAddress, From 0b121d1bff719ee411ba2fb5c2079f0ad7abcd74 Mon Sep 17 00:00:00 2001 From: Lasse Alm Date: Wed, 4 Dec 2024 11:23:06 +0100 Subject: [PATCH 10/12] remove comments --- backend-rust/src/graphql_api.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/backend-rust/src/graphql_api.rs b/backend-rust/src/graphql_api.rs index 383b07b44..8f85dfe1c 100644 --- a/backend-rust/src/graphql_api.rs +++ b/backend-rust/src/graphql_api.rs @@ -5026,10 +5026,8 @@ pub struct TransferMemo { #[ComplexObject] impl TransferMemo { async fn decoded(&self) -> ApiResult { - // Attempt to decode the hex string let decoded_data = hex::decode(&self.raw_hex) .map_err(|e| { - // Log an error if the decoding fails error!("Invalid hex encoding {:?} in a controlled environment", e); ApiError::InternalError("Failed to decode hex data".to_string()) })?; From 9f4911a82de8e2fb42e43548c00306b6e1e7abc1 Mon Sep 17 00:00:00 2001 From: Lasse Alm Date: Wed, 4 Dec 2024 11:23:37 +0100 Subject: [PATCH 11/12] ensured decoding works --- backend-rust/src/graphql_api.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/backend-rust/src/graphql_api.rs b/backend-rust/src/graphql_api.rs index 8f85dfe1c..8595cb78b 100644 --- a/backend-rust/src/graphql_api.rs +++ b/backend-rust/src/graphql_api.rs @@ -4924,10 +4924,8 @@ pub struct DataRegistered { #[ComplexObject] impl DataRegistered { async fn decoded(&self) -> ApiResult { - // Attempt to decode the hex string let decoded_data = hex::decode(&self.data_as_hex) .map_err(|e| { - // Log an error if the decoding fails error!("Invalid hex encoding {:?} in a controlled environment", e); ApiError::InternalError("Failed to decode hex data".to_string()) })?; From 31921d5c6568a2c880b49ac7a98e9b0146f37333 Mon Sep 17 00:00:00 2001 From: Lasse Alm Date: Wed, 4 Dec 2024 11:30:55 +0100 Subject: [PATCH 12/12] fmt --- backend-rust/src/graphql_api.rs | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/backend-rust/src/graphql_api.rs b/backend-rust/src/graphql_api.rs index 8595cb78b..06dfb6dc3 100644 --- a/backend-rust/src/graphql_api.rs +++ b/backend-rust/src/graphql_api.rs @@ -4918,17 +4918,16 @@ pub struct CredentialsUpdated { #[derive(SimpleObject, serde::Serialize, serde::Deserialize)] #[graphql(complex)] pub struct DataRegistered { - data_as_hex: String + data_as_hex: String, } #[ComplexObject] impl DataRegistered { async fn decoded(&self) -> ApiResult { - let decoded_data = hex::decode(&self.data_as_hex) - .map_err(|e| { - error!("Invalid hex encoding {:?} in a controlled environment", e); - ApiError::InternalError("Failed to decode hex data".to_string()) - })?; + let decoded_data = hex::decode(&self.data_as_hex).map_err(|e| { + error!("Invalid hex encoding {:?} in a controlled environment", e); + ApiError::InternalError("Failed to decode hex data".to_string()) + })?; Ok(DecodedText::from_bytes(decoded_data.as_slice())) } @@ -5024,11 +5023,10 @@ pub struct TransferMemo { #[ComplexObject] impl TransferMemo { async fn decoded(&self) -> ApiResult { - let decoded_data = hex::decode(&self.raw_hex) - .map_err(|e| { - error!("Invalid hex encoding {:?} in a controlled environment", e); - ApiError::InternalError("Failed to decode hex data".to_string()) - })?; + let decoded_data = hex::decode(&self.raw_hex).map_err(|e| { + error!("Invalid hex encoding {:?} in a controlled environment", e); + ApiError::InternalError("Failed to decode hex data".to_string()) + })?; Ok(DecodedText::from_bytes(decoded_data.as_slice())) }