From 03a72be09e105c4ce48cc7edda221981cd1ba039 Mon Sep 17 00:00:00 2001 From: Miguel Naveira <47919901+mrnaveira@users.noreply.github.com> Date: Wed, 19 Jun 2024 11:57:25 +0100 Subject: [PATCH] feat(indexer): new JRPC method to list substates (#1049) Description --- * Added new columns `template_address`, `module_name` and `timestamp` to the `substates` SQLite table. * Added new column `timestamp` to the `events` SQLite table. For now it's populated but not used by any method yet. * The `event_scanner` fetches and stores in database the values for all the new fields in substates and events * New JPRC method `list_substates` * Updated TypeScript bindings related with the new indexer method * Removed old logic for event storing in the `event_manager`, it's all now done only on the `event_scanner` and controlled by the event filter in the indexer config Motivation and Context --- On https://github.com/tari-project/tari-dan/pull/1043 we refactored the Indexer to automatically store all substates related to events. This PR follows on that work and adds a new JRPC method `list_substates` that allows to fetch and filter all the substates stored in the Indexer database. It also adds some metadata (`template_address`, `module_name`, `timestamp`) to substates. The new method JRPC method `list_substates` has the following _optional_ parameters: * **filter_by_template**: String with the template address of the components that we want to filter * **filter_by_type**: String with the only type of substate that we want. Possible vaules are `"Component"`, `"Resource"`, `"Vault"`, `"UnclaimedConfidentialOutput"`, `"NonFungible",` `"TransactionReceipt"` and `"FeeClaim"` * **limit**: u64 value for the maximum number of results that we want (intended for pagination) * **offset**: u64 with the position of the first result that we want (intended for pagination) How Has This Been Tested? --- Manually by: * Spawning a local network using `tari_swarm` * Performing some transactions to generate events and substates * Inspecting the Indexer database and also querying the new `list_substate` method What process can a PR reviewer use to test or verify this change? --- See previous section Breaking Changes --- - [x] None - [ ] Requires data directory to be deleted - [ ] Other - Please specify --- .../tari_indexer/src/event_manager.rs | 18 +--- .../tari_indexer/src/event_scanner.rs | 62 +++++++++---- .../tari_indexer/src/graphql/model/events.rs | 2 + .../tari_indexer/src/json_rpc/handlers.rs | 91 ++++++++++++------- .../tari_indexer/src/json_rpc/server.rs | 1 + .../tari_indexer/src/substate_manager.rs | 16 +++- .../down.sql | 1 + .../up.sql | 19 ++++ .../substate_storage_sqlite/models/events.rs | 2 + .../models/substate.rs | 17 ++-- .../src/substate_storage_sqlite/schema.rs | 6 +- .../sqlite_substate_store_factory.rs | 80 ++++++++++++++-- .../tari-indexer-client/ListSubstateItem.ts | 10 ++ .../ListSubstatesRequest.ts | 9 ++ .../ListSubstatesResponse.ts | 6 ++ .../types/tari-indexer-client/SubstateType.ts | 10 ++ bindings/tari-indexer-client.ts | 4 + clients/tari_indexer_client/src/types.rs | 71 +++++++++++++++ integration_tests/src/indexer.rs | 14 ++- 19 files changed, 353 insertions(+), 86 deletions(-) create mode 100644 applications/tari_indexer/src/substate_storage_sqlite/migrations/2024-06-18-095000_add_metadata_to_events_and_substates/down.sql create mode 100644 applications/tari_indexer/src/substate_storage_sqlite/migrations/2024-06-18-095000_add_metadata_to_events_and_substates/up.sql create mode 100644 bindings/src/types/tari-indexer-client/ListSubstateItem.ts create mode 100644 bindings/src/types/tari-indexer-client/ListSubstatesRequest.ts create mode 100644 bindings/src/types/tari-indexer-client/ListSubstatesResponse.ts create mode 100644 bindings/src/types/tari-indexer-client/SubstateType.ts diff --git a/applications/tari_indexer/src/event_manager.rs b/applications/tari_indexer/src/event_manager.rs index 0cda76534..2d4652b7f 100644 --- a/applications/tari_indexer/src/event_manager.rs +++ b/applications/tari_indexer/src/event_manager.rs @@ -75,6 +75,7 @@ impl EventManager { topic: String, payload: &Metadata, version: u64, + timestamp: u64, ) -> Result<(), anyhow::Error> { let mut tx = self.substate_store.create_write_tx()?; let new_event = NewEvent { @@ -84,6 +85,7 @@ impl EventManager { topic, payload: payload.to_json().expect("Failed to convert to JSON"), version: version as i32, + timestamp: timestamp as i64, }; tx.save_event(new_event)?; tx.commit()?; @@ -165,24 +167,10 @@ impl EventManager { .substate_scanner .get_events_for_substate(&substate_id, Some(version)) .await?; - - // stores the newest network events to the db // because the same substate_id with different version // can be processed in the same transaction, we need to avoid // duplicates - for (version, event) in network_events { - let template_address = event.template_address(); - let tx_hash = TransactionId::new(event.tx_hash().into_array()); - let topic = event.topic(); - let payload = event.payload(); - self.save_event_to_db( - &substate_id, - template_address, - tx_hash, - topic, - payload, - u64::from(version), - )?; + for (_, event) in network_events { events.push(event); } diff --git a/applications/tari_indexer/src/event_scanner.rs b/applications/tari_indexer/src/event_scanner.rs index 18e6b00a9..de896eb33 100644 --- a/applications/tari_indexer/src/event_scanner.rs +++ b/applications/tari_indexer/src/event_scanner.rs @@ -20,10 +20,7 @@ // WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE // USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -use std::{ - collections::{HashMap, HashSet}, - str::FromStr, -}; +use std::{collections::HashMap, str::FromStr}; use futures::StreamExt; use log::*; @@ -36,7 +33,7 @@ use tari_dan_storage::consensus_models::{Block, BlockId, Decision, TransactionRe use tari_engine_types::{ commit_result::{ExecuteResult, TransactionResult}, events::Event, - substate::{Substate, SubstateId}, + substate::{Substate, SubstateId, SubstateValue}, }; use tari_epoch_manager::EpochManagerReader; use tari_template_lib::models::{EntityId, TemplateAddress}; @@ -90,6 +87,12 @@ impl TryFrom for EventFilter { } } +#[derive(Default, Debug, Clone, PartialEq, Eq)] +struct TransactionMetadata { + pub transaction_id: TransactionId, + pub timestamp: u64, +} + pub struct EventScanner { network: Network, epoch_manager: Box>, @@ -124,8 +127,6 @@ impl EventScanner { let mut event_count = 0; let current_epoch = self.epoch_manager.current_epoch().await?; - // let network_committee_info = self.epoch_manager.get_network_committees().await?; - // let epoch = network_committee_info.epoch; let current_committees = self.epoch_manager.get_committees(current_epoch).await?; for (shard, mut committee) in current_committees { info!( @@ -134,7 +135,6 @@ impl EventScanner { current_epoch, shard ); - // TODO: use the latest block id that we scanned for each committee let new_blocks = self .get_new_blocks_from_committee(shard, &mut committee, current_epoch) .await?; @@ -143,16 +143,16 @@ impl EventScanner { "Scanned {} blocks", new_blocks.len() ); - let transaction_ids = self.extract_transaction_ids_from_blocks(new_blocks); + let transactions = self.extract_transactions_from_blocks(new_blocks); info!( target: LOG_TARGET, "Scanned {} transactions", - transaction_ids.len() + transactions.len() ); - for transaction_id in transaction_ids { + for transaction in transactions { // fetch all the events in the transaction - let events = self.get_events_for_transaction(transaction_id).await?; + let events = self.get_events_for_transaction(transaction.transaction_id).await?; event_count += events.len(); // only keep the events specified by the indexer filter @@ -163,7 +163,7 @@ impl EventScanner { "Filtered events: {}", filtered_events.len() ); - self.store_events_in_db(&filtered_events).await?; + self.store_events_in_db(&filtered_events, transaction).await?; } } @@ -223,7 +223,11 @@ impl EventScanner { } } - async fn store_events_in_db(&self, events_data: &Vec) -> Result<(), anyhow::Error> { + async fn store_events_in_db( + &self, + events_data: &Vec, + transaction: TransactionMetadata, + ) -> Result<(), anyhow::Error> { let mut tx = self.substate_store.create_write_tx()?; for data in events_data { @@ -234,6 +238,7 @@ impl EventScanner { payload: data.event.payload().to_json().expect("Failed to convert to JSON"), substate_id: data.event.substate_id().map(|s| s.to_string()), version: 0_i32, + timestamp: transaction.timestamp as i64, }; // TODO: properly avoid or handle duplicated events @@ -258,10 +263,16 @@ impl EventScanner { // store/update the related substate if any if let (Some(substate_id), Some(substate)) = (data.event.substate_id(), &data.substate) { + let template_address = Self::extract_template_address_from_substate(substate).map(|t| t.to_string()); + let module_name = Self::extract_module_name_from_substate(substate); let substate_row = NewSubstate { address: substate_id.to_string(), version: i64::from(substate.version()), data: Self::encode_substate(substate)?, + tx_hash: data.event.tx_hash().to_string(), + template_address, + module_name, + timestamp: transaction.timestamp as i64, }; info!( target: LOG_TARGET, @@ -277,6 +288,20 @@ impl EventScanner { Ok(()) } + fn extract_template_address_from_substate(substate: &Substate) -> Option { + match substate.substate_value() { + SubstateValue::Component(c) => Some(c.template_address), + _ => None, + } + } + + fn extract_module_name_from_substate(substate: &Substate) -> Option { + match substate.substate_value() { + SubstateValue::Component(c) => Some(c.module_name.to_owned()), + _ => None, + } + } + fn encode_substate(substate: &Substate) -> Result { let pretty_json = serde_json::to_string_pretty(&substate)?; Ok(pretty_json) @@ -373,11 +398,14 @@ impl EventScanner { } } - fn extract_transaction_ids_from_blocks(&self, blocks: Vec) -> HashSet { + fn extract_transactions_from_blocks(&self, blocks: Vec) -> Vec { blocks .iter() - .flat_map(|b| b.all_accepted_transactions_ids()) - .copied() + .flat_map(|b| b.all_accepted_transactions_ids().map(|id| (id, b.timestamp()))) + .map(|(transaction_id, timestamp)| TransactionMetadata { + transaction_id: *transaction_id, + timestamp, + }) .collect() } diff --git a/applications/tari_indexer/src/graphql/model/events.rs b/applications/tari_indexer/src/graphql/model/events.rs index d9400b8ce..cad455881 100644 --- a/applications/tari_indexer/src/graphql/model/events.rs +++ b/applications/tari_indexer/src/graphql/model/events.rs @@ -166,6 +166,7 @@ impl EventQuery { topic: String, payload: String, version: u64, + timestamp: u64, ) -> Result { info!( target: LOG_TARGET, @@ -185,6 +186,7 @@ impl EventQuery { topic.clone(), &payload, version, + timestamp, )?; Ok(Event { diff --git a/applications/tari_indexer/src/json_rpc/handlers.rs b/applications/tari_indexer/src/json_rpc/handlers.rs index 9817496d8..2e5074f7f 100644 --- a/applications/tari_indexer/src/json_rpc/handlers.rs +++ b/applications/tari_indexer/src/json_rpc/handlers.rs @@ -44,41 +44,41 @@ use tari_dan_engine::{template::TemplateModuleLoader, wasm::WasmModule}; use tari_dan_p2p::TariMessagingSpec; use tari_dan_storage::consensus_models::Decision; use tari_epoch_manager::{base_layer::EpochManagerHandle, EpochManagerReader}; -use tari_indexer_client::{ - types, - types::{ - AddPeerRequest, - AddPeerResponse, - ConnectionDirection, - GetAllVnsRequest, - GetAllVnsResponse, - GetCommsStatsResponse, - GetConnectionsResponse, - GetEpochManagerStatsResponse, - GetIdentityResponse, - GetNonFungibleCollectionsResponse, - GetNonFungibleCountRequest, - GetNonFungibleCountResponse, - GetNonFungiblesRequest, - GetNonFungiblesResponse, - GetRelatedTransactionsRequest, - GetRelatedTransactionsResponse, - GetSubstateRequest, - GetSubstateResponse, - GetTemplateDefinitionRequest, - GetTemplateDefinitionResponse, - GetTransactionResultRequest, - GetTransactionResultResponse, - IndexerTransactionFinalizedResult, - InspectSubstateRequest, - InspectSubstateResponse, - ListTemplatesRequest, - ListTemplatesResponse, - NonFungibleSubstate, - SubmitTransactionRequest, - SubmitTransactionResponse, - TemplateMetadata, - }, +use tari_indexer_client::types::{ + self, + AddPeerRequest, + AddPeerResponse, + ConnectionDirection, + GetAllVnsRequest, + GetAllVnsResponse, + GetCommsStatsResponse, + GetConnectionsResponse, + GetEpochManagerStatsResponse, + GetIdentityResponse, + GetNonFungibleCollectionsResponse, + GetNonFungibleCountRequest, + GetNonFungibleCountResponse, + GetNonFungiblesRequest, + GetNonFungiblesResponse, + GetRelatedTransactionsRequest, + GetRelatedTransactionsResponse, + GetSubstateRequest, + GetSubstateResponse, + GetTemplateDefinitionRequest, + GetTemplateDefinitionResponse, + GetTransactionResultRequest, + GetTransactionResultResponse, + IndexerTransactionFinalizedResult, + InspectSubstateRequest, + InspectSubstateResponse, + ListSubstatesRequest, + ListSubstatesResponse, + ListTemplatesRequest, + ListTemplatesResponse, + NonFungibleSubstate, + SubmitTransactionRequest, + SubmitTransactionResponse, + TemplateMetadata, }; use tari_networking::{is_supported_multiaddr, NetworkingHandle, NetworkingService}; use tari_validator_node_rpc::client::{SubstateResult, TariValidatorNodeRpcClientFactory, TransactionResultStatus}; @@ -261,6 +261,27 @@ impl JsonRpcHandlers { })) } + pub async fn list_substates(&self, value: JsonRpcExtractor) -> JrpcResult { + let answer_id = value.get_answer_id(); + let ListSubstatesRequest { + filter_by_template, + filter_by_type, + limit, + offset, + } = value.parse_params()?; + + let substates = self + .substate_manager + .list_substates(filter_by_type, filter_by_template, limit, offset) + .await + .map_err(|e| { + warn!(target: LOG_TARGET, "Error getting substate: {}", e); + Self::internal_error(answer_id, format!("Error getting substate: {}", e)) + })?; + + Ok(JsonRpcResponse::success(answer_id, ListSubstatesResponse { substates })) + } + pub async fn get_substate(&self, value: JsonRpcExtractor) -> JrpcResult { let answer_id = value.get_answer_id(); let request: GetSubstateRequest = value.parse_params()?; diff --git a/applications/tari_indexer/src/json_rpc/server.rs b/applications/tari_indexer/src/json_rpc/server.rs index 138a70296..6ff39f556 100644 --- a/applications/tari_indexer/src/json_rpc/server.rs +++ b/applications/tari_indexer/src/json_rpc/server.rs @@ -63,6 +63,7 @@ async fn handler(Extension(handlers): Extension>, value: Js "get_all_vns" => handlers.get_all_vns(value).await, "add_peer" => handlers.add_peer(value).await, "get_comms_stats" => handlers.get_comms_stats(value).await, + "list_substates" => handlers.list_substates(value).await, "get_substate" => handlers.get_substate(value).await, "inspect_substate" => handlers.inspect_substate(value).await, "get_connections" => handlers.get_connections(value).await, diff --git a/applications/tari_indexer/src/substate_manager.rs b/applications/tari_indexer/src/substate_manager.rs index 1af2fc337..a7ad36475 100644 --- a/applications/tari_indexer/src/substate_manager.rs +++ b/applications/tari_indexer/src/substate_manager.rs @@ -28,7 +28,9 @@ use tari_dan_app_utilities::substate_file_cache::SubstateFileCache; use tari_dan_common_types::PeerAddress; use tari_engine_types::substate::{Substate, SubstateId}; use tari_epoch_manager::base_layer::EpochManagerHandle; +use tari_indexer_client::types::{ListSubstateItem, SubstateType}; use tari_indexer_lib::{substate_scanner::SubstateScanner, NonFungibleSubstate}; +use tari_template_lib::models::TemplateAddress; use tari_transaction::TransactionId; use tari_validator_node_rpc::client::{SubstateResult, TariValidatorNodeRpcClientFactory}; @@ -88,12 +90,24 @@ impl SubstateManager { } } + pub async fn list_substates( + &self, + filter_by_type: Option, + filter_by_template: Option, + limit: Option, + offset: Option, + ) -> Result, anyhow::Error> { + let mut tx = self.substate_store.create_read_tx()?; + let substates = tx.list_substates(filter_by_type, filter_by_template, limit, offset)?; + Ok(substates) + } + pub async fn get_substate( &self, substate_address: &SubstateId, version: Option, ) -> Result, anyhow::Error> { - // we store the latest version of the substates in the watchlist, + // we store the latest version of the substates related to the events // so we will return the substate directly from database if it's there if let Some(substate) = self.get_substate_from_db(substate_address, version).await? { return Ok(Some(substate)); diff --git a/applications/tari_indexer/src/substate_storage_sqlite/migrations/2024-06-18-095000_add_metadata_to_events_and_substates/down.sql b/applications/tari_indexer/src/substate_storage_sqlite/migrations/2024-06-18-095000_add_metadata_to_events_and_substates/down.sql new file mode 100644 index 000000000..d9a93fe9a --- /dev/null +++ b/applications/tari_indexer/src/substate_storage_sqlite/migrations/2024-06-18-095000_add_metadata_to_events_and_substates/down.sql @@ -0,0 +1 @@ +-- This file should undo anything in `up.sql` diff --git a/applications/tari_indexer/src/substate_storage_sqlite/migrations/2024-06-18-095000_add_metadata_to_events_and_substates/up.sql b/applications/tari_indexer/src/substate_storage_sqlite/migrations/2024-06-18-095000_add_metadata_to_events_and_substates/up.sql new file mode 100644 index 000000000..1bc1b64cd --- /dev/null +++ b/applications/tari_indexer/src/substate_storage_sqlite/migrations/2024-06-18-095000_add_metadata_to_events_and_substates/up.sql @@ -0,0 +1,19 @@ +-- The transaction hash column will be similar to the one in the "events" table +alter table substates + drop column transaction_hash; +alter table substates + add column tx_hash text not NULL; + +-- Tempalte address for component substates +alter table substates + add column template_address text NULL; + +-- Name of the template module for components +alter table substates + add column module_name text NULL; + +-- block timestamp for events and substates +alter table events + add column timestamp bigint not NULL; +alter table substates + add column timestamp bigint not NULL; \ No newline at end of file diff --git a/applications/tari_indexer/src/substate_storage_sqlite/models/events.rs b/applications/tari_indexer/src/substate_storage_sqlite/models/events.rs index 9a13d3046..f98d9bf37 100644 --- a/applications/tari_indexer/src/substate_storage_sqlite/models/events.rs +++ b/applications/tari_indexer/src/substate_storage_sqlite/models/events.rs @@ -40,6 +40,7 @@ pub struct Event { pub payload: String, pub version: i32, pub substate_id: Option, + pub timestamp: i64, } #[derive(Debug, Clone, Insertable, AsChangeset)] @@ -52,6 +53,7 @@ pub struct NewEvent { pub payload: String, pub version: i32, pub substate_id: Option, + pub timestamp: i64, } #[derive(Debug, Clone, Insertable, AsChangeset)] diff --git a/applications/tari_indexer/src/substate_storage_sqlite/models/substate.rs b/applications/tari_indexer/src/substate_storage_sqlite/models/substate.rs index 8c399b669..dd7f09580 100644 --- a/applications/tari_indexer/src/substate_storage_sqlite/models/substate.rs +++ b/applications/tari_indexer/src/substate_storage_sqlite/models/substate.rs @@ -36,23 +36,22 @@ pub struct Substate { pub address: String, pub version: i64, pub data: String, - pub transaction_hash: Option>, + pub tx_hash: String, + pub template_address: Option, + pub module_name: Option, + pub timestamp: i64, } impl TryFrom for SubstateResponse { type Error = anyhow::Error; fn try_from(row: SubstateRow) -> Result { - let tx_hash = if let Some(hash) = row.transaction_hash { - hash.try_into()? - } else { - TransactionId::default() - }; + let created_by_transaction = TransactionId::from_hex(&row.tx_hash)?; Ok(SubstateResponse { address: row.address.parse()?, version: row.version.try_into()?, substate: serde_json::from_str(&row.data)?, - created_by_transaction: tx_hash, + created_by_transaction, }) } } @@ -63,4 +62,8 @@ pub struct NewSubstate { pub address: String, pub version: i64, pub data: String, + pub tx_hash: String, + pub template_address: Option, + pub module_name: Option, + pub timestamp: i64, } diff --git a/applications/tari_indexer/src/substate_storage_sqlite/schema.rs b/applications/tari_indexer/src/substate_storage_sqlite/schema.rs index 2ff7abca5..1b1feca87 100644 --- a/applications/tari_indexer/src/substate_storage_sqlite/schema.rs +++ b/applications/tari_indexer/src/substate_storage_sqlite/schema.rs @@ -15,7 +15,10 @@ diesel::table! { address -> Text, version -> BigInt, data -> Text, - transaction_hash -> Nullable, + tx_hash -> Text, + template_address -> Nullable, + module_name -> Nullable, + timestamp -> BigInt, } } @@ -28,6 +31,7 @@ diesel::table! { payload -> Text, version -> Integer, substate_id -> Nullable, + timestamp -> BigInt, } } diff --git a/applications/tari_indexer/src/substate_storage_sqlite/sqlite_substate_store_factory.rs b/applications/tari_indexer/src/substate_storage_sqlite/sqlite_substate_store_factory.rs index 40f758787..cf42dce0f 100644 --- a/applications/tari_indexer/src/substate_storage_sqlite/sqlite_substate_store_factory.rs +++ b/applications/tari_indexer/src/substate_storage_sqlite/sqlite_substate_store_factory.rs @@ -25,6 +25,7 @@ use std::{ fs::create_dir_all, ops::{Deref, DerefMut}, path::PathBuf, + str::FromStr, sync::{Arc, Mutex}, }; @@ -35,12 +36,14 @@ use diesel::{ sql_types::{Integer, Nullable, Text}, SqliteConnection, }; -use diesel_migrations::EmbeddedMigrations; +use diesel_migrations::{EmbeddedMigrations, MigrationHarness}; use log::*; use tari_dan_common_types::{shard::Shard, Epoch}; use tari_dan_storage::{consensus_models::BlockId, StorageError}; use tari_dan_storage_sqlite::{error::SqliteStorageError, SqliteTransaction}; use tari_engine_types::substate::SubstateId; +use tari_indexer_client::types::{ListSubstateItem, SubstateType}; +use tari_template_lib::models::TemplateAddress; use tari_transaction::TransactionId; use thiserror::Error; @@ -48,12 +51,9 @@ use super::models::{ events::{EventData, NewEvent, NewScannedBlockId}, non_fungible_index::{IndexedNftSubstate, NewNonFungibleIndex}, }; -use crate::{ - diesel_migrations::MigrationHarness, - substate_storage_sqlite::models::{ - events::{Event, NewEventPayloadField, ScannedBlockId}, - substate::{NewSubstate, Substate}, - }, +use crate::substate_storage_sqlite::models::{ + events::{Event, NewEventPayloadField, ScannedBlockId}, + substate::{NewSubstate, Substate}, }; const LOG_TARGET: &str = "tari::indexer::substate_storage_sqlite"; @@ -178,6 +178,13 @@ impl<'a> SqliteSubstateStoreReadTransaction<'a> { } pub trait SubstateStoreReadTransaction { + fn list_substates( + &mut self, + filter_by_type: Option, + filter_by_template: Option, + limit: Option, + offset: Option, + ) -> Result, StorageError>; fn get_substate(&mut self, address: &SubstateId) -> Result, StorageError>; fn get_latest_version_for_substate(&mut self, address: &SubstateId) -> Result, StorageError>; fn get_all_addresses(&mut self) -> Result, StorageError>; @@ -221,6 +228,65 @@ pub trait SubstateStoreReadTransaction { } impl SubstateStoreReadTransaction for SqliteSubstateStoreReadTransaction<'_> { + fn list_substates( + &mut self, + by_type: Option, + by_template_address: Option, + limit: Option, + offset: Option, + ) -> Result, StorageError> { + use crate::substate_storage_sqlite::schema::substates; + + let mut query = substates::table.into_boxed(); + + if let Some(template_address) = by_template_address { + query = query.filter(substates::template_address.eq(template_address.to_string())); + } + + if let Some(substate_type) = by_type { + let address_like = match substate_type { + SubstateType::NonFungible => format!("resource_% {}_%", substate_type.as_prefix_str()), + _ => format!("{}_%", substate_type.as_prefix_str()), + }; + query = query.filter(substates::address.like(address_like)); + } + + if let Some(limit) = limit { + query = query.limit(limit as i64); + } + if let Some(offset) = offset { + query = query.offset(offset as i64); + } + + let substates: Vec = query + .get_results(self.connection()) + .map_err(|e| StorageError::QueryError { + reason: format!("list_substates: {}", e), + })?; + + let items = substates + .into_iter() + .map(|s| { + let substate_id = SubstateId::from_str(&s.address)?; + let version = u32::try_from(s.version)?; + let template_address = s.template_address.map(|h| TemplateAddress::from_hex(&h)).transpose()?; + let timestamp = u64::try_from(s.timestamp)?; + Ok(ListSubstateItem { + substate_id, + module_name: s.module_name, + version, + template_address, + timestamp, + }) + }) + .collect::, anyhow::Error>>() + .map_err(|e| StorageError::QueryError { + reason: format!("list_substates: invalid substate items: {}", e), + })?; + + Ok(items) + } + fn get_substate(&mut self, address: &SubstateId) -> Result, StorageError> { use crate::substate_storage_sqlite::schema::substates; diff --git a/bindings/src/types/tari-indexer-client/ListSubstateItem.ts b/bindings/src/types/tari-indexer-client/ListSubstateItem.ts new file mode 100644 index 000000000..56e60ded7 --- /dev/null +++ b/bindings/src/types/tari-indexer-client/ListSubstateItem.ts @@ -0,0 +1,10 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { SubstateId } from "../SubstateId"; + +export interface ListSubstateItem { + substate_id: SubstateId; + module_name: string | null; + version: number; + template_address: string | null; + timestamp: bigint; +} diff --git a/bindings/src/types/tari-indexer-client/ListSubstatesRequest.ts b/bindings/src/types/tari-indexer-client/ListSubstatesRequest.ts new file mode 100644 index 000000000..6adf828ca --- /dev/null +++ b/bindings/src/types/tari-indexer-client/ListSubstatesRequest.ts @@ -0,0 +1,9 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { SubstateType } from "./SubstateType"; + +export interface ListSubstatesRequest { + filter_by_template: string | null; + filter_by_type: SubstateType | null; + limit: bigint | null; + offset: bigint | null; +} diff --git a/bindings/src/types/tari-indexer-client/ListSubstatesResponse.ts b/bindings/src/types/tari-indexer-client/ListSubstatesResponse.ts new file mode 100644 index 000000000..81931d454 --- /dev/null +++ b/bindings/src/types/tari-indexer-client/ListSubstatesResponse.ts @@ -0,0 +1,6 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { ListSubstateItem } from "./ListSubstateItem"; + +export interface ListSubstatesResponse { + substates: Array; +} diff --git a/bindings/src/types/tari-indexer-client/SubstateType.ts b/bindings/src/types/tari-indexer-client/SubstateType.ts new file mode 100644 index 000000000..72e7c5a85 --- /dev/null +++ b/bindings/src/types/tari-indexer-client/SubstateType.ts @@ -0,0 +1,10 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type SubstateType = + | "Component" + | "Resource" + | "Vault" + | "UnclaimedConfidentialOutput" + | "NonFungible" + | "TransactionReceipt" + | "FeeClaim"; diff --git a/bindings/tari-indexer-client.ts b/bindings/tari-indexer-client.ts index f1263f110..4d3719552 100644 --- a/bindings/tari-indexer-client.ts +++ b/bindings/tari-indexer-client.ts @@ -5,6 +5,7 @@ export * from "./src/types/tari-indexer-client/GetAllVnsRequest"; export * from "./src/types/tari-indexer-client/NonFungibleSubstate"; export * from "./src/types/tari-indexer-client/GetTransactionResultResponse"; export * from "./src/types/tari-indexer-client/GetNonFungibleCountResponse"; +export * from "./src/types/tari-indexer-client/ListSubstatesRequest"; export * from "./src/types/tari-indexer-client/GetAllVnsResponse"; export * from "./src/types/tari-indexer-client/GetConnectionsResponse"; export * from "./src/types/tari-indexer-client/GetTemplateDefinitionRequest"; @@ -20,6 +21,7 @@ export * from "./src/types/tari-indexer-client/GetNonFungibleCountRequest"; export * from "./src/types/tari-indexer-client/GetTransactionResultRequest"; export * from "./src/types/tari-indexer-client/GetRelatedTransactionsResponse"; export * from "./src/types/tari-indexer-client/GetNonFungiblesResponse"; +export * from "./src/types/tari-indexer-client/SubstateType"; export * from "./src/types/tari-indexer-client/GetRelatedTransactionsRequest"; export * from "./src/types/tari-indexer-client/GetEpochManagerStatsResponse"; export * from "./src/types/tari-indexer-client/AddPeerRequest"; @@ -27,8 +29,10 @@ export * from "./src/types/tari-indexer-client/GetIdentityResponse"; export * from "./src/types/tari-indexer-client/ListTemplatesResponse"; export * from "./src/types/tari-indexer-client/SubmitTransactionResponse"; export * from "./src/types/tari-indexer-client/GetCommsStatsResponse"; +export * from "./src/types/tari-indexer-client/ListSubstatesResponse"; export * from "./src/types/tari-indexer-client/GetSubstateResponse"; export * from "./src/types/tari-indexer-client/IndexerTransactionFinalizedResult"; export * from "./src/types/tari-indexer-client/GetNonFungibleCollectionsResponse"; export * from "./src/types/tari-indexer-client/GetSubstateRequest"; export * from "./src/types/tari-indexer-client/SubmitTransactionRequest"; +export * from "./src/types/tari-indexer-client/ListSubstateItem"; diff --git a/clients/tari_indexer_client/src/types.rs b/clients/tari_indexer_client/src/types.rs index fc7817dbf..623d30e4d 100644 --- a/clients/tari_indexer_client/src/types.rs +++ b/clients/tari_indexer_client/src/types.rs @@ -22,6 +22,77 @@ use tari_transaction::{SubstateRequirement, Transaction, TransactionId}; #[cfg(feature = "ts")] use ts_rs::TS; +#[derive(Debug, Clone, Copy, Deserialize, Serialize)] +#[cfg_attr( + feature = "ts", + derive(ts_rs::TS), + ts(export, export_to = "../../bindings/src/types/tari-indexer-client/") +)] +pub enum SubstateType { + Component, + Resource, + Vault, + UnclaimedConfidentialOutput, + NonFungible, + TransactionReceipt, + FeeClaim, +} + +impl SubstateType { + pub fn as_prefix_str(&self) -> &str { + match self { + SubstateType::Component => "component", + SubstateType::Resource => "resource", + SubstateType::Vault => "vault", + SubstateType::UnclaimedConfidentialOutput => "commitment", + SubstateType::NonFungible => "nft", + SubstateType::TransactionReceipt => "txreceipt", + SubstateType::FeeClaim => "feeclaim", + } + } +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[cfg_attr( + feature = "ts", + derive(TS), + ts(export, export_to = "../../bindings/src/types/tari-indexer-client/") +)] +pub struct ListSubstatesRequest { + #[serde(default, deserialize_with = "serde_tools::string::option::deserialize")] + #[cfg_attr(feature = "ts", ts(type = "string | null"))] + pub filter_by_template: Option, + pub filter_by_type: Option, + pub limit: Option, + pub offset: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[cfg_attr( + feature = "ts", + derive(TS), + ts(export, export_to = "../../bindings/src/types/tari-indexer-client/") +)] +pub struct ListSubstatesResponse { + pub substates: Vec, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[cfg_attr( + feature = "ts", + derive(TS), + ts(export, export_to = "../../bindings/src/types/tari-indexer-client/") +)] +pub struct ListSubstateItem { + pub substate_id: SubstateId, + pub module_name: Option, + pub version: u32, + #[serde(default, with = "serde_tools::string::option")] + #[cfg_attr(feature = "ts", ts(type = "string | null"))] + pub template_address: Option, + pub timestamp: u64, +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[cfg_attr( feature = "ts", diff --git a/integration_tests/src/indexer.rs b/integration_tests/src/indexer.rs index 308fd7528..53e75d138 100644 --- a/integration_tests/src/indexer.rs +++ b/integration_tests/src/indexer.rs @@ -20,7 +20,11 @@ // WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE // USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -use std::{collections::HashMap, path::PathBuf, time::Duration}; +use std::{ + collections::HashMap, + path::PathBuf, + time::{Duration, SystemTime}, +}; use reqwest::Url; use tari_common::{ @@ -109,10 +113,14 @@ impl IndexerProcess { let payload = HashMap::::from([("my".to_string(), "event".to_string())]) .to_json() .unwrap(); + let timestamp = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_millis(); let query = format!( "{{ saveEvent(substateId: {:?}, templateAddress: {:?}, txHash: {:?}, topic: {:?}, payload: {:?}, version: \ - {:?}) {{ substateId templateAddress txHash topic payload }} }}", - substate_id, template_address, tx_hash, topic, payload, version + {:?}, timestamp: {}) {{ substateId templateAddress txHash topic payload }} }}", + substate_id, template_address, tx_hash, topic, payload, version, timestamp ); let res = graphql_client .send_request::>(&query, None, None)