From 95f82089eebd9e6ca25076952c334004f8ec688c Mon Sep 17 00:00:00 2001 From: incrypto32 Date: Thu, 11 Jul 2024 15:18:36 +0530 Subject: [PATCH 01/29] graph: Add support for subgraph datasource in manifest --- core/src/subgraph/context/instance/mod.rs | 47 +-- core/src/subgraph/instance_manager.rs | 2 +- core/src/subgraph/registrar.rs | 1 - graph/src/blockchain/mod.rs | 1 + graph/src/components/subgraph/instance.rs | 1 + graph/src/data/subgraph/api_version.rs | 3 + graph/src/data/subgraph/mod.rs | 14 +- graph/src/data_source/mod.rs | 83 +++++ graph/src/data_source/subgraph.rs | 305 ++++++++++++++++++ runtime/wasm/src/host.rs | 2 + runtime/wasm/src/module/mod.rs | 1 + .../tests/chain/ethereum/manifest.rs | 46 ++- .../subgraph-data-sources/abis/Contract.abi | 15 + .../subgraph-data-sources/package.json | 13 + .../subgraph-data-sources/schema.graphql | 6 + .../subgraph-data-sources/src/mapping.ts | 15 + .../subgraph-data-sources/subgraph.yaml | 19 ++ tests/runner-tests/yarn.lock | 98 +++++- tests/tests/runner_tests.rs | 18 ++ 19 files changed, 657 insertions(+), 33 deletions(-) create mode 100644 graph/src/data_source/subgraph.rs create mode 100644 tests/runner-tests/subgraph-data-sources/abis/Contract.abi create mode 100644 tests/runner-tests/subgraph-data-sources/package.json create mode 100644 tests/runner-tests/subgraph-data-sources/schema.graphql create mode 100644 tests/runner-tests/subgraph-data-sources/src/mapping.ts create mode 100644 tests/runner-tests/subgraph-data-sources/subgraph.yaml diff --git a/core/src/subgraph/context/instance/mod.rs b/core/src/subgraph/context/instance/mod.rs index ed242836a28..5a805f34095 100644 --- a/core/src/subgraph/context/instance/mod.rs +++ b/core/src/subgraph/context/instance/mod.rs @@ -138,35 +138,38 @@ where ); } - let is_onchain = data_source.is_onchain(); let Some(host) = self.new_host(logger.clone(), data_source)? else { return Ok(None); }; // Check for duplicates and add the host. - if is_onchain { - // `onchain_hosts` will remain ordered by the creation block. - // See also 8f1bca33-d3b7-4035-affc-fd6161a12448. - ensure!( - self.onchain_hosts - .last() - .and_then(|h| h.creation_block_number()) - <= host.data_source().creation_block(), - ); - - if self.onchain_hosts.contains(&host) { - Ok(None) - } else { - self.onchain_hosts.push(host.cheap_clone()); - Ok(Some(host)) + match host.data_source() { + DataSource::Onchain(_) => { + // `onchain_hosts` will remain ordered by the creation block. + // See also 8f1bca33-d3b7-4035-affc-fd6161a12448. + ensure!( + self.onchain_hosts + .last() + .and_then(|h| h.creation_block_number()) + <= host.data_source().creation_block(), + ); + + if self.onchain_hosts.contains(&host) { + Ok(None) + } else { + self.onchain_hosts.push(host.cheap_clone()); + Ok(Some(host)) + } } - } else { - if self.offchain_hosts.contains(&host) { - Ok(None) - } else { - self.offchain_hosts.push(host.cheap_clone()); - Ok(Some(host)) + DataSource::Offchain(_) => { + if self.offchain_hosts.contains(&host) { + Ok(None) + } else { + self.offchain_hosts.push(host.cheap_clone()); + Ok(Some(host)) + } } + DataSource::Subgraph(_) => Ok(None), } } diff --git a/core/src/subgraph/instance_manager.rs b/core/src/subgraph/instance_manager.rs index c98641539d9..75b0b86f81f 100644 --- a/core/src/subgraph/instance_manager.rs +++ b/core/src/subgraph/instance_manager.rs @@ -318,7 +318,7 @@ impl SubgraphInstanceManager { let start_blocks: Vec = data_sources .iter() - .filter_map(|d| d.as_onchain().map(|d: &C::DataSource| d.start_block())) + .filter_map(|d| d.start_block()) .collect(); let end_blocks: BTreeSet = manifest diff --git a/core/src/subgraph/registrar.rs b/core/src/subgraph/registrar.rs index fe80d118457..e8ad83315e0 100644 --- a/core/src/subgraph/registrar.rs +++ b/core/src/subgraph/registrar.rs @@ -629,7 +629,6 @@ async fn create_subgraph_version( ) .map_err(SubgraphRegistrarError::ResolveError) .await?; - // Determine if the graft_base should be validated. // Validate the graft_base if there is a pending graft, ensuring its presence. // If the subgraph is new (indicated by DeploymentNotFound), the graft_base should be validated. diff --git a/graph/src/blockchain/mod.rs b/graph/src/blockchain/mod.rs index 73cac816728..1b897440b9b 100644 --- a/graph/src/blockchain/mod.rs +++ b/graph/src/blockchain/mod.rs @@ -489,6 +489,7 @@ impl FromStr for BlockchainKind { "cosmos" => Ok(BlockchainKind::Cosmos), "substreams" => Ok(BlockchainKind::Substreams), "starknet" => Ok(BlockchainKind::Starknet), + "subgraph" => Ok(BlockchainKind::Ethereum), // TODO(krishna): We should detect the blockchain kind from the source subgraph _ => Err(anyhow!("unknown blockchain kind {}", s)), } } diff --git a/graph/src/components/subgraph/instance.rs b/graph/src/components/subgraph/instance.rs index 470e50334d3..889690c3916 100644 --- a/graph/src/components/subgraph/instance.rs +++ b/graph/src/components/subgraph/instance.rs @@ -20,6 +20,7 @@ impl From<&DataSourceTemplate> for InstanceDSTemplate { match value { DataSourceTemplate::Onchain(ds) => Self::Onchain(ds.info()), DataSourceTemplate::Offchain(ds) => Self::Offchain(ds.clone()), + DataSourceTemplate::Subgraph(_) => todo!(), // TODO(krishna) } } } diff --git a/graph/src/data/subgraph/api_version.rs b/graph/src/data/subgraph/api_version.rs index e626e9f1dbc..43ee639007c 100644 --- a/graph/src/data/subgraph/api_version.rs +++ b/graph/src/data/subgraph/api_version.rs @@ -54,6 +54,9 @@ pub const SPEC_VERSION_1_1_0: Version = Version::new(1, 1, 0); // Enables eth call declarations and indexed arguments(topics) filtering in manifest pub const SPEC_VERSION_1_2_0: Version = Version::new(1, 2, 0); +// Enables subgraphs as datasource +pub const SPEC_VERSION_1_3_0: Version = Version::new(1, 3, 0); + // The latest spec version available pub const LATEST_VERSION: &Version = &SPEC_VERSION_1_2_0; diff --git a/graph/src/data/subgraph/mod.rs b/graph/src/data/subgraph/mod.rs index 52b0f4dfed1..941764b9a12 100644 --- a/graph/src/data/subgraph/mod.rs +++ b/graph/src/data/subgraph/mod.rs @@ -33,7 +33,7 @@ use web3::types::Address; use crate::{ bail, - blockchain::{BlockPtr, Blockchain, DataSource as _}, + blockchain::{BlockPtr, Blockchain}, components::{ link_resolver::LinkResolver, store::{StoreError, SubgraphStore}, @@ -140,6 +140,10 @@ impl DeploymentHash { link: format!("/ipfs/{}", self), } } + + pub fn to_bytes(&self) -> Vec { + self.0.as_bytes().to_vec() + } } impl Deref for DeploymentHash { @@ -719,7 +723,7 @@ impl UnvalidatedSubgraphManifest { .0 .data_sources .iter() - .filter_map(|d| Some(d.as_onchain()?.network()?.to_string())) + .filter_map(|d| Some(d.network()?.to_string())) .collect::>(); networks.sort(); networks.dedup(); @@ -765,11 +769,9 @@ impl SubgraphManifest { max_spec_version: semver::Version, ) -> Result { let unresolved = UnresolvedSubgraphManifest::parse(id, raw)?; - let resolved = unresolved .resolve(resolver, logger, max_spec_version) .await?; - Ok(resolved) } @@ -777,14 +779,14 @@ impl SubgraphManifest { // Assume the manifest has been validated, ensuring network names are homogenous self.data_sources .iter() - .find_map(|d| Some(d.as_onchain()?.network()?.to_string())) + .find_map(|d| Some(d.network()?.to_string())) .expect("Validated manifest does not have a network defined on any datasource") } pub fn start_blocks(&self) -> Vec { self.data_sources .iter() - .filter_map(|d| Some(d.as_onchain()?.start_block())) + .filter_map(|d| d.start_block()) .collect() } diff --git a/graph/src/data_source/mod.rs b/graph/src/data_source/mod.rs index a38148b25fe..38a166710e0 100644 --- a/graph/src/data_source/mod.rs +++ b/graph/src/data_source/mod.rs @@ -1,6 +1,8 @@ pub mod causality_region; pub mod offchain; +pub mod subgraph; +pub use self::DataSource as DataSourceEnum; pub use causality_region::CausalityRegion; #[cfg(test)] @@ -17,6 +19,7 @@ use crate::{ store::{BlockNumber, StoredDynamicDataSource}, }, data_source::offchain::OFFCHAIN_KINDS, + data_source::subgraph::SUBGRAPH_DS_KIND, prelude::{CheapClone as _, DataSourceContext}, schema::{EntityType, InputSchema}, }; @@ -35,6 +38,7 @@ use thiserror::Error; pub enum DataSource { Onchain(C::DataSource), Offchain(offchain::DataSource), + Subgraph(subgraph::DataSource), } #[derive(Error, Debug)] @@ -89,6 +93,23 @@ impl DataSource { match self { Self::Onchain(ds) => Some(ds), Self::Offchain(_) => None, + Self::Subgraph(_) => None, + } + } + + pub fn as_subgraph(&self) -> Option<&subgraph::DataSource> { + match self { + Self::Onchain(_) => None, + Self::Offchain(_) => None, + Self::Subgraph(ds) => Some(ds), + } + } + + pub fn is_chain_based(&self) -> bool { + match self { + Self::Onchain(_) => true, + Self::Offchain(_) => false, + Self::Subgraph(_) => true, } } @@ -96,6 +117,23 @@ impl DataSource { match self { Self::Onchain(_) => None, Self::Offchain(ds) => Some(ds), + Self::Subgraph(_) => None, + } + } + + pub fn network(&self) -> Option<&str> { + match self { + DataSourceEnum::Onchain(ds) => ds.network(), + DataSourceEnum::Offchain(_) => None, + DataSourceEnum::Subgraph(ds) => ds.network(), + } + } + + pub fn start_block(&self) -> Option { + match self { + DataSourceEnum::Onchain(ds) => Some(ds.start_block()), + DataSourceEnum::Offchain(_) => None, + DataSourceEnum::Subgraph(ds) => Some(ds.source.start_block), } } @@ -111,6 +149,7 @@ impl DataSource { match self { Self::Onchain(ds) => ds.address().map(ToOwned::to_owned), Self::Offchain(ds) => ds.address(), + Self::Subgraph(ds) => ds.address(), } } @@ -118,6 +157,7 @@ impl DataSource { match self { Self::Onchain(ds) => ds.name(), Self::Offchain(ds) => &ds.name, + Self::Subgraph(ds) => &ds.name, } } @@ -125,6 +165,7 @@ impl DataSource { match self { Self::Onchain(ds) => ds.kind().to_owned(), Self::Offchain(ds) => ds.kind.to_string(), + Self::Subgraph(ds) => ds.kind.clone(), } } @@ -132,6 +173,7 @@ impl DataSource { match self { Self::Onchain(ds) => ds.min_spec_version(), Self::Offchain(ds) => ds.min_spec_version(), + Self::Subgraph(ds) => ds.min_spec_version(), } } @@ -139,6 +181,7 @@ impl DataSource { match self { Self::Onchain(ds) => ds.end_block(), Self::Offchain(_) => None, + Self::Subgraph(_) => None, } } @@ -146,6 +189,7 @@ impl DataSource { match self { Self::Onchain(ds) => ds.creation_block(), Self::Offchain(ds) => ds.creation_block, + Self::Subgraph(ds) => ds.creation_block, } } @@ -153,6 +197,7 @@ impl DataSource { match self { Self::Onchain(ds) => ds.context(), Self::Offchain(ds) => ds.context.clone(), + Self::Subgraph(ds) => ds.context.clone(), } } @@ -160,6 +205,7 @@ impl DataSource { match self { Self::Onchain(ds) => ds.api_version(), Self::Offchain(ds) => ds.mapping.api_version.clone(), + Self::Subgraph(ds) => ds.mapping.api_version.clone(), } } @@ -167,6 +213,7 @@ impl DataSource { match self { Self::Onchain(ds) => ds.runtime(), Self::Offchain(ds) => Some(ds.mapping.runtime.cheap_clone()), + Self::Subgraph(ds) => Some(ds.mapping.runtime.cheap_clone()), } } @@ -176,6 +223,7 @@ impl DataSource { // been enforced. Self::Onchain(_) => EntityTypeAccess::Any, Self::Offchain(ds) => EntityTypeAccess::Restriced(ds.mapping.entities.clone()), + Self::Subgraph(_) => EntityTypeAccess::Any, } } @@ -183,6 +231,7 @@ impl DataSource { match self { Self::Onchain(ds) => ds.handler_kinds(), Self::Offchain(ds) => vec![ds.handler_kind()].into_iter().collect(), + Self::Subgraph(ds) => vec![ds.handler_kind()].into_iter().collect(), } } @@ -190,6 +239,7 @@ impl DataSource { match self { Self::Onchain(ds) => ds.has_declared_calls(), Self::Offchain(_) => false, + Self::Subgraph(_) => false, } } @@ -209,6 +259,7 @@ impl DataSource { } (Self::Onchain(_), TriggerData::Offchain(_)) | (Self::Offchain(_), TriggerData::Onchain(_)) => Ok(None), + (Self::Subgraph(_), _) => todo!(), // TODO(krishna) } } @@ -224,6 +275,7 @@ impl DataSource { match self { Self::Onchain(ds) => ds.as_stored_dynamic_data_source(), Self::Offchain(ds) => ds.as_stored_dynamic_data_source(), + Self::Subgraph(_) => todo!(), // TODO(krishna) } } @@ -240,6 +292,7 @@ impl DataSource { offchain::DataSource::from_stored_dynamic_data_source(template, stored) .map(DataSource::Offchain) } + DataSourceTemplate::Subgraph(_) => todo!(), // TODO(krishna) } } @@ -247,6 +300,7 @@ impl DataSource { match self { Self::Onchain(ds) => ds.validate(spec_version), Self::Offchain(_) => vec![], + Self::Subgraph(_) => vec![], // TODO(krishna) } } @@ -254,6 +308,7 @@ impl DataSource { match self { Self::Onchain(_) => CausalityRegion::ONCHAIN, Self::Offchain(ds) => ds.causality_region, + Self::Subgraph(_) => CausalityRegion::ONCHAIN, } } } @@ -262,6 +317,7 @@ impl DataSource { pub enum UnresolvedDataSource { Onchain(C::UnresolvedDataSource), Offchain(offchain::UnresolvedDataSource), + Subgraph(subgraph::UnresolvedDataSource), } impl UnresolvedDataSource { @@ -276,6 +332,10 @@ impl UnresolvedDataSource { .resolve(resolver, logger, manifest_idx) .await .map(DataSource::Onchain), + Self::Subgraph(unresolved) => unresolved + .resolve(resolver, logger, manifest_idx) + .await + .map(DataSource::Subgraph), Self::Offchain(_unresolved) => { anyhow::bail!( "static file data sources are not yet supported, \\ @@ -299,6 +359,7 @@ pub struct DataSourceTemplateInfo { pub enum DataSourceTemplate { Onchain(C::DataSourceTemplate), Offchain(offchain::DataSourceTemplate), + Subgraph(subgraph::DataSourceTemplate), } impl DataSourceTemplate { @@ -306,6 +367,7 @@ impl DataSourceTemplate { match self { DataSourceTemplate::Onchain(template) => template.info(), DataSourceTemplate::Offchain(template) => template.clone().into(), + DataSourceTemplate::Subgraph(template) => template.clone().into(), } } @@ -313,6 +375,7 @@ impl DataSourceTemplate { match self { Self::Onchain(ds) => Some(ds), Self::Offchain(_) => None, + Self::Subgraph(_) => todo!(), // TODO(krishna) } } @@ -320,6 +383,7 @@ impl DataSourceTemplate { match self { Self::Onchain(_) => None, Self::Offchain(t) => Some(t), + Self::Subgraph(_) => todo!(), // TODO(krishna) } } @@ -327,6 +391,7 @@ impl DataSourceTemplate { match self { Self::Onchain(ds) => Some(ds), Self::Offchain(_) => None, + Self::Subgraph(_) => todo!(), // TODO(krishna) } } @@ -334,6 +399,7 @@ impl DataSourceTemplate { match self { Self::Onchain(ds) => &ds.name(), Self::Offchain(ds) => &ds.name, + Self::Subgraph(ds) => &ds.name, } } @@ -341,6 +407,7 @@ impl DataSourceTemplate { match self { Self::Onchain(ds) => ds.api_version(), Self::Offchain(ds) => ds.mapping.api_version.clone(), + Self::Subgraph(ds) => ds.mapping.api_version.clone(), } } @@ -348,6 +415,7 @@ impl DataSourceTemplate { match self { Self::Onchain(ds) => ds.runtime(), Self::Offchain(ds) => Some(ds.mapping.runtime.clone()), + Self::Subgraph(ds) => Some(ds.mapping.runtime.clone()), } } @@ -355,6 +423,7 @@ impl DataSourceTemplate { match self { Self::Onchain(ds) => ds.manifest_idx(), Self::Offchain(ds) => ds.manifest_idx, + Self::Subgraph(ds) => ds.manifest_idx, } } @@ -362,6 +431,7 @@ impl DataSourceTemplate { match self { Self::Onchain(ds) => ds.kind().to_string(), Self::Offchain(ds) => ds.kind.to_string(), + Self::Subgraph(ds) => ds.kind.clone(), } } } @@ -370,6 +440,7 @@ impl DataSourceTemplate { pub enum UnresolvedDataSourceTemplate { Onchain(C::UnresolvedDataSourceTemplate), Offchain(offchain::UnresolvedDataSourceTemplate), + Subgraph(subgraph::UnresolvedDataSourceTemplate), } impl Default for UnresolvedDataSourceTemplate { @@ -395,6 +466,10 @@ impl UnresolvedDataSourceTemplate { .resolve(resolver, logger, manifest_idx, schema) .await .map(DataSourceTemplate::Offchain), + Self::Subgraph(ds) => ds + .resolve(resolver, logger, manifest_idx) + .await + .map(DataSourceTemplate::Subgraph), } } } @@ -490,6 +565,7 @@ impl TriggerData { pub enum MappingTrigger { Onchain(C::MappingTrigger), Offchain(offchain::TriggerData), + Subgraph(subgraph::TriggerData), } impl MappingTrigger { @@ -497,6 +573,7 @@ impl MappingTrigger { match self { Self::Onchain(trigger) => Some(trigger.error_context()), Self::Offchain(_) => None, // TODO: Add error context for offchain triggers + Self::Subgraph(_) => None, // TODO(krishna) } } @@ -504,6 +581,7 @@ impl MappingTrigger { match self { Self::Onchain(trigger) => Some(trigger), Self::Offchain(_) => None, + Self::Subgraph(_) => todo!(), // TODO(krishna) } } } @@ -515,6 +593,7 @@ macro_rules! clone_data_source { match self { Self::Onchain(ds) => Self::Onchain(ds.clone()), Self::Offchain(ds) => Self::Offchain(ds.clone()), + Self::Subgraph(ds) => Self::Subgraph(ds.clone()), } } } @@ -541,6 +620,10 @@ macro_rules! deserialize_data_source { offchain::$t::deserialize(map.into_deserializer()) .map_err(serde::de::Error::custom) .map($t::Offchain) + } else if SUBGRAPH_DS_KIND == kind { + subgraph::$t::deserialize(map.into_deserializer()) + .map_err(serde::de::Error::custom) + .map($t::Subgraph) } else if (&C::KIND.to_string() == kind) || C::ALIASES.contains(&kind) { C::$t::deserialize(map.into_deserializer()) .map_err(serde::de::Error::custom) diff --git a/graph/src/data_source/subgraph.rs b/graph/src/data_source/subgraph.rs new file mode 100644 index 00000000000..dba43786438 --- /dev/null +++ b/graph/src/data_source/subgraph.rs @@ -0,0 +1,305 @@ +use crate::{ + blockchain::{Block, Blockchain}, + components::{ + link_resolver::LinkResolver, + store::{BlockNumber, Entity}, + }, + data::{subgraph::SPEC_VERSION_1_3_0, value::Word}, + data_source, + prelude::{DataSourceContext, DeploymentHash, Link}, +}; +use anyhow::{Context, Error}; +use serde::Deserialize; +use slog::{info, Logger}; +use std::{fmt, sync::Arc}; + +use super::{DataSourceTemplateInfo, TriggerWithHandler}; + +pub const SUBGRAPH_DS_KIND: &str = "subgraph"; + +const ENTITY_HANDLER_KINDS: &str = "entity"; + +#[derive(Debug, Clone)] +pub struct DataSource { + pub kind: String, + pub name: String, + pub network: String, + pub manifest_idx: u32, + pub source: Source, + pub mapping: Mapping, + pub context: Arc>, + pub creation_block: Option, +} + +impl DataSource { + pub fn new( + kind: String, + name: String, + network: String, + manifest_idx: u32, + source: Source, + mapping: Mapping, + context: Arc>, + creation_block: Option, + ) -> Self { + Self { + kind, + name, + network, + manifest_idx, + source, + mapping, + context, + creation_block, + } + } + + pub fn min_spec_version(&self) -> semver::Version { + SPEC_VERSION_1_3_0 + } + + pub fn handler_kind(&self) -> &str { + ENTITY_HANDLER_KINDS + } + + pub fn network(&self) -> Option<&str> { + Some(&self.network) + } + + pub fn match_and_decode( + &self, + block: &Arc, + trigger: &TriggerData, + ) -> Option>> { + if self.source.address != trigger.source { + return None; + } + + let trigger_ref = self.mapping.handlers.iter().find_map(|handler| { + if handler.entity != trigger.entity_type { + return None; + } + + Some(TriggerWithHandler::new( + data_source::MappingTrigger::Subgraph(trigger.clone()), + handler.handler.clone(), + block.ptr(), + block.timestamp(), + )) + }); + + return trigger_ref; + } + + pub fn address(&self) -> Option> { + Some(self.source.address().to_bytes()) + } + + pub fn source_subgraph(&self) -> DeploymentHash { + self.source.address() + } +} + +pub type Base64 = Word; + +#[derive(Clone, Debug, Default, Hash, Eq, PartialEq, Deserialize)] +pub struct Source { + pub address: DeploymentHash, + #[serde(default)] + pub start_block: BlockNumber, +} + +impl Source { + /// The concept of an address may or not make sense for a subgraph data source, but graph node + /// will use this in a few places where some sort of not necessarily unique id is useful: + /// 1. This is used as the value to be returned to mappings from the `dataSource.address()` host + /// function, so changing this is a breaking change. + /// 2. This is used to match with triggers with hosts in `fn hosts_for_trigger`, so make sure + /// the `source` of the data source is equal the `source` of the `TriggerData`. + pub fn address(&self) -> DeploymentHash { + self.address.clone() + } +} + +#[derive(Clone, Debug)] +pub struct Mapping { + pub language: String, + pub api_version: semver::Version, + pub entities: Vec, + pub handlers: Vec, + pub runtime: Arc>, + pub link: Link, +} + +#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] +pub struct EntityHandler { + pub handler: String, + pub entity: String, +} + +#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize)] +pub struct UnresolvedDataSource { + pub kind: String, + pub name: String, + pub network: String, + pub source: UnresolvedSource, + pub mapping: UnresolvedMapping, +} + +#[derive(Clone, Debug, Default, Hash, Eq, PartialEq, Deserialize)] +pub struct UnresolvedSource { + address: DeploymentHash, + #[serde(default)] + start_block: BlockNumber, +} + +#[derive(Clone, Debug, Default, Hash, Eq, PartialEq, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct UnresolvedMapping { + pub api_version: String, + pub language: String, + pub file: Link, + pub handlers: Vec, + pub entities: Vec, +} + +impl UnresolvedDataSource { + #[allow(dead_code)] + pub(super) async fn resolve( + self, + resolver: &Arc, + logger: &Logger, + manifest_idx: u32, + ) -> Result { + info!(logger, "Resolve subgraph data source"; + "name" => &self.name, + "kind" => &self.kind, + "source" => format_args!("{:?}", &self.source), + ); + + let kind = self.kind; + let source = Source { + address: self.source.address, + start_block: self.source.start_block, + }; + + Ok(DataSource { + manifest_idx, + kind, + name: self.name, + network: self.network, + source, + mapping: self.mapping.resolve(resolver, logger).await?, + context: Arc::new(None), + creation_block: None, + }) + } +} + +impl UnresolvedMapping { + pub async fn resolve( + self, + resolver: &Arc, + logger: &Logger, + ) -> Result { + info!(logger, "Resolve subgraph ds mapping"; "link" => &self.file.link); + + Ok(Mapping { + language: self.language, + api_version: semver::Version::parse(&self.api_version)?, + entities: self.entities, + handlers: self.handlers, + runtime: Arc::new(resolver.cat(logger, &self.file).await?), + link: self.file, + }) + } +} + +#[derive(Clone, Debug, Deserialize)] +pub struct UnresolvedDataSourceTemplate { + pub kind: String, + pub network: Option, + pub name: String, + pub mapping: UnresolvedMapping, +} + +#[derive(Clone, Debug)] +pub struct DataSourceTemplate { + pub kind: String, + pub network: Option, + pub name: String, + pub manifest_idx: u32, + pub mapping: Mapping, +} + +impl Into for DataSourceTemplate { + fn into(self) -> DataSourceTemplateInfo { + let DataSourceTemplate { + kind, + network: _, + name, + manifest_idx, + mapping, + } = self; + + DataSourceTemplateInfo { + api_version: mapping.api_version.clone(), + runtime: Some(mapping.runtime), + name, + manifest_idx: Some(manifest_idx), + kind: kind.to_string(), + } + } +} + +impl UnresolvedDataSourceTemplate { + pub async fn resolve( + self, + resolver: &Arc, + logger: &Logger, + manifest_idx: u32, + ) -> Result { + let kind = self.kind; + + let mapping = self + .mapping + .resolve(resolver, logger) + .await + .with_context(|| format!("failed to resolve data source template {}", self.name))?; + + Ok(DataSourceTemplate { + kind, + network: self.network, + name: self.name, + manifest_idx, + mapping, + }) + } +} + +#[derive(Clone, PartialEq, Eq)] +pub struct TriggerData { + pub source: DeploymentHash, + pub entity: Entity, + pub entity_type: String, +} + +impl TriggerData { + pub fn new(source: DeploymentHash, entity: Entity, entity_type: String) -> Self { + Self { + source, + entity, + entity_type, + } + } +} + +impl fmt::Debug for TriggerData { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!( + f, + "TriggerData {{ source: {:?}, entity: {:?} }}", + self.source, self.entity, + ) + } +} diff --git a/runtime/wasm/src/host.rs b/runtime/wasm/src/host.rs index 3ecee7ba753..ebf107fb3ec 100644 --- a/runtime/wasm/src/host.rs +++ b/runtime/wasm/src/host.rs @@ -366,6 +366,7 @@ impl RuntimeHostTrait for RuntimeHost { match self.data_source() { DataSource::Onchain(_) => None, DataSource::Offchain(ds) => ds.done_at(), + DataSource::Subgraph(_) => None, } } @@ -373,6 +374,7 @@ impl RuntimeHostTrait for RuntimeHost { match self.data_source() { DataSource::Onchain(_) => {} DataSource::Offchain(ds) => ds.set_done_at(block), + DataSource::Subgraph(_) => {} } } diff --git a/runtime/wasm/src/module/mod.rs b/runtime/wasm/src/module/mod.rs index ffe4f7aba8e..ee19cd173aa 100644 --- a/runtime/wasm/src/module/mod.rs +++ b/runtime/wasm/src/module/mod.rs @@ -81,6 +81,7 @@ where match self { MappingTrigger::Onchain(trigger) => trigger.to_asc_ptr(heap, gas), MappingTrigger::Offchain(trigger) => trigger.to_asc_ptr(heap, gas), + MappingTrigger::Subgraph(_) => todo!(), // TODO(krishna) } } } diff --git a/store/test-store/tests/chain/ethereum/manifest.rs b/store/test-store/tests/chain/ethereum/manifest.rs index 9089ec4f572..34eaf110f77 100644 --- a/store/test-store/tests/chain/ethereum/manifest.rs +++ b/store/test-store/tests/chain/ethereum/manifest.rs @@ -11,10 +11,10 @@ use graph::data::store::Value; use graph::data::subgraph::schema::SubgraphError; use graph::data::subgraph::{ Prune, LATEST_VERSION, SPEC_VERSION_0_0_4, SPEC_VERSION_0_0_7, SPEC_VERSION_0_0_8, - SPEC_VERSION_0_0_9, SPEC_VERSION_1_0_0, SPEC_VERSION_1_2_0, + SPEC_VERSION_0_0_9, SPEC_VERSION_1_0_0, SPEC_VERSION_1_2_0, SPEC_VERSION_1_3_0, }; use graph::data_source::offchain::OffchainDataSourceKind; -use graph::data_source::DataSourceTemplate; +use graph::data_source::{DataSourceEnum, DataSourceTemplate}; use graph::entity; use graph::env::ENV_VARS; use graph::prelude::web3::types::H256; @@ -166,10 +166,52 @@ specVersion: 0.0.7 let data_source = match &manifest.templates[0] { DataSourceTemplate::Offchain(ds) => ds, DataSourceTemplate::Onchain(_) => unreachable!(), + DataSourceTemplate::Subgraph(_) => unreachable!(), }; assert_eq!(data_source.kind, OffchainDataSourceKind::Ipfs); } +#[tokio::test] +async fn subgraph_ds_manifest() { + let yaml = " +schema: + file: + /: /ipfs/Qmschema +dataSources: + - name: SubgraphSource + kind: subgraph + entities: + - Gravatar + network: mainnet + source: + address: 'QmUVaWpdKgcxBov1jHEa8dr46d2rkVzfHuZFu4fXJ4sFse' + startBlock: 0 + mapping: + apiVersion: 0.0.6 + language: wasm/assemblyscript + entities: + - TestEntity + file: + /: /ipfs/Qmmapping + handlers: + - handler: handleEntity + entity: User +specVersion: 1.3.0 +"; + + let manifest = resolve_manifest(yaml, SPEC_VERSION_1_3_0).await; + + assert_eq!("Qmmanifest", manifest.id.as_str()); + assert_eq!(manifest.data_sources.len(), 1); + let data_source = &manifest.data_sources[0]; + match data_source { + DataSourceEnum::Subgraph(ds) => { + assert_eq!(ds.name, "SubgraphSource"); + } + _ => panic!("Expected a subgraph data source"), + } +} + #[tokio::test] async fn graft_manifest() { const YAML: &str = " diff --git a/tests/runner-tests/subgraph-data-sources/abis/Contract.abi b/tests/runner-tests/subgraph-data-sources/abis/Contract.abi new file mode 100644 index 00000000000..9d9f56b9263 --- /dev/null +++ b/tests/runner-tests/subgraph-data-sources/abis/Contract.abi @@ -0,0 +1,15 @@ +[ + { + "anonymous": false, + "inputs": [ + { + "indexed": false, + "internalType": "string", + "name": "testCommand", + "type": "string" + } + ], + "name": "TestEvent", + "type": "event" + } +] diff --git a/tests/runner-tests/subgraph-data-sources/package.json b/tests/runner-tests/subgraph-data-sources/package.json new file mode 100644 index 00000000000..87537290ad2 --- /dev/null +++ b/tests/runner-tests/subgraph-data-sources/package.json @@ -0,0 +1,13 @@ +{ + "name": "subgraph-data-sources", + "version": "0.1.0", + "scripts": { + "codegen": "graph codegen --skip-migrations", + "create:test": "graph create test/subgraph-data-sources --node $GRAPH_NODE_ADMIN_URI", + "deploy:test": "graph deploy test/subgraph-data-sources --version-label v0.0.1 --ipfs $IPFS_URI --node $GRAPH_NODE_ADMIN_URI" + }, + "devDependencies": { + "@graphprotocol/graph-cli": "0.79.0-alpha-20240711124603-49edf22", + "@graphprotocol/graph-ts": "0.31.0" + } +} diff --git a/tests/runner-tests/subgraph-data-sources/schema.graphql b/tests/runner-tests/subgraph-data-sources/schema.graphql new file mode 100644 index 00000000000..6f97fa65c43 --- /dev/null +++ b/tests/runner-tests/subgraph-data-sources/schema.graphql @@ -0,0 +1,6 @@ +type Data @entity { + id: ID! + foo: String + bar: Int + isTest: Boolean +} diff --git a/tests/runner-tests/subgraph-data-sources/src/mapping.ts b/tests/runner-tests/subgraph-data-sources/src/mapping.ts new file mode 100644 index 00000000000..3446d1f83c4 --- /dev/null +++ b/tests/runner-tests/subgraph-data-sources/src/mapping.ts @@ -0,0 +1,15 @@ +import { BigInt, dataSource, ethereum, log } from "@graphprotocol/graph-ts"; +import { Data } from "../generated/schema"; + +export function handleBlock(block: ethereum.Block): void { + let foo = dataSource.context().getString("foo"); + let bar = dataSource.context().getI32("bar"); + let isTest = dataSource.context().getBoolean("isTest"); + if (block.number == BigInt.fromI32(0)) { + let data = new Data("0"); + data.foo = foo; + data.bar = bar; + data.isTest = isTest; + data.save(); + } +} diff --git a/tests/runner-tests/subgraph-data-sources/subgraph.yaml b/tests/runner-tests/subgraph-data-sources/subgraph.yaml new file mode 100644 index 00000000000..b1a3fcbb486 --- /dev/null +++ b/tests/runner-tests/subgraph-data-sources/subgraph.yaml @@ -0,0 +1,19 @@ +specVersion: 1.3.0 +schema: + file: ./schema.graphql +dataSources: + - kind: subgraph + name: Contract + network: test + source: + address: 'QmHash' + startBlock: 6082461 + mapping: + apiVersion: 0.0.7 + language: wasm/assemblyscript + entities: + - Gravatar + handlers: + - handler: handleBlock + entity: Gravatar + file: ./src/mapping.ts diff --git a/tests/runner-tests/yarn.lock b/tests/runner-tests/yarn.lock index 50e0c2b471f..9f3bdae834d 100644 --- a/tests/runner-tests/yarn.lock +++ b/tests/runner-tests/yarn.lock @@ -349,6 +349,40 @@ which "2.0.2" yaml "1.10.2" +"@graphprotocol/graph-cli@0.79.0-alpha-20240711124603-49edf22": + version "0.79.0-alpha-20240711124603-49edf22" + resolved "https://registry.yarnpkg.com/@graphprotocol/graph-cli/-/graph-cli-0.79.0-alpha-20240711124603-49edf22.tgz#4e3f6201932a0b68ce64d6badd8432cf2bead3c2" + integrity sha512-fZrdPiFbbbBVMnvsjfKA+j48WzzquaHQIpozBqnUKRPCV1n1NenIaq2nH16mlMwovRIS7AAIVCpa0QYQuPzw7Q== + dependencies: + "@float-capital/float-subgraph-uncrashable" "^0.0.0-alpha.4" + "@oclif/core" "2.8.6" + "@oclif/plugin-autocomplete" "^2.3.6" + "@oclif/plugin-not-found" "^2.4.0" + "@whatwg-node/fetch" "^0.8.4" + assemblyscript "0.19.23" + binary-install-raw "0.0.13" + chalk "3.0.0" + chokidar "3.5.3" + debug "4.3.4" + docker-compose "0.23.19" + dockerode "2.5.8" + fs-extra "9.1.0" + glob "9.3.5" + gluegun "5.1.6" + graphql "15.5.0" + immutable "4.2.1" + ipfs-http-client "55.0.0" + jayson "4.0.0" + js-yaml "3.14.1" + open "8.4.2" + prettier "3.0.3" + semver "7.4.0" + sync-request "6.1.0" + tmp-promise "3.0.3" + web3-eth-abi "1.7.0" + which "2.0.2" + yaml "1.10.2" + "@graphprotocol/graph-ts@0.30.0": version "0.30.0" resolved "https://registry.npmjs.org/@graphprotocol/graph-ts/-/graph-ts-0.30.0.tgz" @@ -1473,6 +1507,11 @@ defaults@^1.0.3: dependencies: clone "^1.0.2" +define-lazy-prop@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/define-lazy-prop/-/define-lazy-prop-2.0.0.tgz#3f7ae421129bcaaac9bc74905c98a0009ec9ee7f" + integrity sha512-Ds09qNh8yw3khSjiJjiUInaGX9xlqZDY7JVryGxdxV7NPeuqQfplOpQ66yJFZut3jLa5zOwkXw1g9EI2uKh4Og== + delay@^5.0.0: version "5.0.0" resolved "https://registry.npmjs.org/delay/-/delay-5.0.0.tgz" @@ -1545,6 +1584,13 @@ ejs@3.1.6: dependencies: jake "^10.6.1" +ejs@3.1.8: + version "3.1.8" + resolved "https://registry.yarnpkg.com/ejs/-/ejs-3.1.8.tgz#758d32910c78047585c7ef1f92f9ee041c1c190b" + integrity sha512-/sXZeMlhS0ArkfX2Aw780gJzXSMPnKjtspYZv+f3NiKLlubezAHDU5+9xz6gd3/NhG3txQCo6xlglmTS+oTGEQ== + dependencies: + jake "^10.8.5" + ejs@^3.1.8: version "3.1.9" resolved "https://registry.npmjs.org/ejs/-/ejs-3.1.9.tgz" @@ -1996,6 +2042,42 @@ gluegun@5.1.2: which "2.0.2" yargs-parser "^21.0.0" +gluegun@5.1.6: + version "5.1.6" + resolved "https://registry.yarnpkg.com/gluegun/-/gluegun-5.1.6.tgz#74ec13193913dc610f5c1a4039972c70c96a7bad" + integrity sha512-9zbi4EQWIVvSOftJWquWzr9gLX2kaDgPkNR5dYWbM53eVvCI3iKuxLlnKoHC0v4uPoq+Kr/+F569tjoFbA4DSA== + dependencies: + apisauce "^2.1.5" + app-module-path "^2.2.0" + cli-table3 "0.6.0" + colors "1.4.0" + cosmiconfig "7.0.1" + cross-spawn "7.0.3" + ejs "3.1.8" + enquirer "2.3.6" + execa "5.1.1" + fs-jetpack "4.3.1" + lodash.camelcase "^4.3.0" + lodash.kebabcase "^4.1.1" + lodash.lowercase "^4.3.0" + lodash.lowerfirst "^4.3.1" + lodash.pad "^4.5.1" + lodash.padend "^4.6.1" + lodash.padstart "^4.6.1" + lodash.repeat "^4.1.0" + lodash.snakecase "^4.1.1" + lodash.startcase "^4.4.0" + lodash.trim "^4.5.1" + lodash.trimend "^4.5.1" + lodash.trimstart "^4.5.1" + lodash.uppercase "^4.3.0" + lodash.upperfirst "^4.3.1" + ora "4.0.2" + pluralize "^8.0.0" + semver "7.3.5" + which "2.0.2" + yargs-parser "^21.0.0" + graceful-fs@^4.1.6, graceful-fs@^4.2.0: version "4.2.11" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.11.tgz#4183e4e8bf08bb6e05bbb2f7d2e0c8f712ca40e3" @@ -2282,7 +2364,7 @@ is-binary-path@~2.1.0: dependencies: binary-extensions "^2.0.0" -is-docker@^2.0.0: +is-docker@^2.0.0, is-docker@^2.1.1: version "2.2.1" resolved "https://registry.npmjs.org/is-docker/-/is-docker-2.2.1.tgz" integrity sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ== @@ -2922,6 +3004,15 @@ onetime@^5.1.0, onetime@^5.1.2: dependencies: mimic-fn "^2.1.0" +open@8.4.2: + version "8.4.2" + resolved "https://registry.yarnpkg.com/open/-/open-8.4.2.tgz#5b5ffe2a8f793dcd2aad73e550cb87b59cb084f9" + integrity sha512-7x81NCL719oNbsq/3mh+hVrAWmFuEYUqrq/Iw3kUzH8ReypT9QQ0BLoJS7/G9k6N81XjW4qHWtjWwe/9eLy1EQ== + dependencies: + define-lazy-prop "^2.0.0" + is-docker "^2.1.1" + is-wsl "^2.2.0" + ora@4.0.2: version "4.0.2" resolved "https://registry.npmjs.org/ora/-/ora-4.0.2.tgz" @@ -3042,6 +3133,11 @@ prettier@1.19.1: resolved "https://registry.npmjs.org/prettier/-/prettier-1.19.1.tgz" integrity sha512-s7PoyDv/II1ObgQunCbB9PdLmUcBZcnWOcxDh7O0N/UwDEsHyqkW+Qh28jW+mVuCdx7gLB0BotYI1Y6uI9iyew== +prettier@3.0.3: + version "3.0.3" + resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.0.3.tgz#432a51f7ba422d1469096c0fdc28e235db8f9643" + integrity sha512-L/4pUDMxcNa8R/EthV08Zt42WBO4h1rarVtK0K+QJG0X187OLo7l699jWw0GKuwzkPQ//jMFA/8Xm6Fh3J/DAg== + process-nextick-args@~2.0.0: version "2.0.1" resolved "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz" diff --git a/tests/tests/runner_tests.rs b/tests/tests/runner_tests.rs index 7da707ac7cd..169836f2390 100644 --- a/tests/tests/runner_tests.rs +++ b/tests/tests/runner_tests.rs @@ -1077,6 +1077,24 @@ async fn parse_data_source_context() { ); } +#[tokio::test] +async fn subgraph_data_sources() { + let RunnerTestRecipe { stores, test_info } = + RunnerTestRecipe::new("subgraph-data-sources", "subgraph-data-sources").await; + + let blocks = { + let block_0 = genesis(); + let block_1 = empty_block(block_0.ptr(), test_ptr(1)); + let block_2 = empty_block(block_1.ptr(), test_ptr(2)); + vec![block_0, block_1, block_2] + }; + let stop_block = blocks.last().unwrap().block.ptr(); + let chain = chain(&test_info.test_name, blocks, &stores, None).await; + + let ctx = fixture::setup(&test_info, &stores, &chain, None, None).await; + ctx.start_and_sync_to(stop_block).await; +} + #[tokio::test] async fn retry_create_ds() { let RunnerTestRecipe { stores, test_info } = From 243027e4f5c93ed95753c0441843cf90c4afa3b5 Mon Sep 17 00:00:00 2001 From: incrypto32 Date: Fri, 12 Jul 2024 16:03:47 +0530 Subject: [PATCH 02/29] graph: wrap TriggerFilter with TriggerFilterWrapper --- chain/arweave/src/chain.rs | 9 ++++-- chain/cosmos/src/chain.rs | 9 ++++-- chain/ethereum/src/chain.rs | 9 +++--- chain/near/src/chain.rs | 8 ++--- chain/starknet/src/chain.rs | 6 ++-- chain/substreams/src/chain.rs | 6 ++-- core/src/subgraph/context/mod.rs | 4 +-- core/src/subgraph/runner.rs | 17 ++++++---- core/src/subgraph/stream.rs | 6 ++-- graph/src/blockchain/mock.rs | 53 +++++++++++++++++++++++--------- graph/src/blockchain/mod.rs | 18 +++++++++-- tests/tests/runner_tests.rs | 10 +++--- 12 files changed, 102 insertions(+), 53 deletions(-) diff --git a/chain/arweave/src/chain.rs b/chain/arweave/src/chain.rs index 8d40408a463..70c03d832dd 100644 --- a/chain/arweave/src/chain.rs +++ b/chain/arweave/src/chain.rs @@ -3,7 +3,7 @@ use graph::blockchain::client::ChainClient; use graph::blockchain::firehose_block_ingestor::FirehoseBlockIngestor; use graph::blockchain::{ BasicBlockchainBuilder, Block, BlockIngestor, BlockchainBuilder, BlockchainKind, - EmptyNodeCapabilities, NoopDecoderHook, NoopRuntimeAdapter, + EmptyNodeCapabilities, NoopDecoderHook, NoopRuntimeAdapter, TriggerFilterWrapper, }; use graph::cheap_clone::CheapClone; use graph::components::adapter::ChainId; @@ -119,7 +119,7 @@ impl Blockchain for Chain { deployment: DeploymentLocator, store: impl DeploymentCursorTracker, start_blocks: Vec, - filter: Arc, + filter: Arc<&TriggerFilterWrapper>, unified_api_version: UnifiedMappingApiVersion, ) -> Result>, Error> { let adapter = self @@ -135,7 +135,10 @@ impl Blockchain for Chain { .subgraph_logger(&deployment) .new(o!("component" => "FirehoseBlockStream")); - let firehose_mapper = Arc::new(FirehoseMapper { adapter, filter }); + let firehose_mapper = Arc::new(FirehoseMapper { + adapter, + filter: filter.filter.clone(), + }); Ok(Box::new(FirehoseBlockStream::new( deployment.hash, diff --git a/chain/cosmos/src/chain.rs b/chain/cosmos/src/chain.rs index 955aa7efc3c..b21acb3a8e6 100644 --- a/chain/cosmos/src/chain.rs +++ b/chain/cosmos/src/chain.rs @@ -1,5 +1,5 @@ use graph::blockchain::firehose_block_ingestor::FirehoseBlockIngestor; -use graph::blockchain::{BlockIngestor, NoopDecoderHook}; +use graph::blockchain::{BlockIngestor, NoopDecoderHook, TriggerFilterWrapper}; use graph::components::adapter::ChainId; use graph::env::EnvVars; use graph::prelude::MetricsRegistry; @@ -113,7 +113,7 @@ impl Blockchain for Chain { deployment: DeploymentLocator, store: impl DeploymentCursorTracker, start_blocks: Vec, - filter: Arc, + filter: Arc<&TriggerFilterWrapper>, unified_api_version: UnifiedMappingApiVersion, ) -> Result>, Error> { let adapter = self @@ -129,7 +129,10 @@ impl Blockchain for Chain { .subgraph_logger(&deployment) .new(o!("component" => "FirehoseBlockStream")); - let firehose_mapper = Arc::new(FirehoseMapper { adapter, filter }); + let firehose_mapper = Arc::new(FirehoseMapper { + adapter, + filter: filter.filter.clone(), + }); Ok(Box::new(FirehoseBlockStream::new( deployment.hash, diff --git a/chain/ethereum/src/chain.rs b/chain/ethereum/src/chain.rs index 1def8c483cc..458265c9b87 100644 --- a/chain/ethereum/src/chain.rs +++ b/chain/ethereum/src/chain.rs @@ -3,7 +3,8 @@ use anyhow::{Context, Error}; use graph::blockchain::client::ChainClient; use graph::blockchain::firehose_block_ingestor::{FirehoseBlockIngestor, Transforms}; use graph::blockchain::{ - BlockIngestor, BlockTime, BlockchainKind, ChainIdentifier, TriggersAdapterSelector, + BlockIngestor, BlockTime, BlockchainKind, ChainIdentifier, TriggerFilterWrapper, + TriggersAdapterSelector, }; use graph::components::adapter::ChainId; use graph::components::store::DeploymentCursorTracker; @@ -409,7 +410,7 @@ impl Blockchain for Chain { deployment: DeploymentLocator, store: impl DeploymentCursorTracker, start_blocks: Vec, - filter: Arc, + filter: Arc<&TriggerFilterWrapper>, unified_api_version: UnifiedMappingApiVersion, ) -> Result>, Error> { let current_ptr = store.block_ptr(); @@ -421,7 +422,7 @@ impl Blockchain for Chain { deployment, start_blocks, current_ptr, - filter, + filter.filter.clone(), unified_api_version, ) .await @@ -434,7 +435,7 @@ impl Blockchain for Chain { store.firehose_cursor(), start_blocks, current_ptr, - filter, + filter.filter.clone(), unified_api_version, ) .await diff --git a/chain/near/src/chain.rs b/chain/near/src/chain.rs index 283552e7f33..962978304a9 100644 --- a/chain/near/src/chain.rs +++ b/chain/near/src/chain.rs @@ -4,7 +4,7 @@ use graph::blockchain::firehose_block_ingestor::FirehoseBlockIngestor; use graph::blockchain::substreams_block_stream::SubstreamsBlockStream; use graph::blockchain::{ BasicBlockchainBuilder, BlockIngestor, BlockchainBuilder, BlockchainKind, NoopDecoderHook, - NoopRuntimeAdapter, + NoopRuntimeAdapter, TriggerFilterWrapper, }; use graph::cheap_clone::CheapClone; use graph::components::adapter::ChainId; @@ -230,7 +230,7 @@ impl Blockchain for Chain { deployment: DeploymentLocator, store: impl DeploymentCursorTracker, start_blocks: Vec, - filter: Arc, + filter: Arc<&TriggerFilterWrapper>, unified_api_version: UnifiedMappingApiVersion, ) -> Result>, Error> { if self.prefer_substreams { @@ -242,7 +242,7 @@ impl Blockchain for Chain { deployment, store.firehose_cursor(), store.block_ptr(), - filter, + filter.filter.clone(), ) .await; } @@ -254,7 +254,7 @@ impl Blockchain for Chain { store.firehose_cursor(), start_blocks, store.block_ptr(), - filter, + filter.filter.clone(), unified_api_version, ) .await diff --git a/chain/starknet/src/chain.rs b/chain/starknet/src/chain.rs index cd10af5f965..442474f507b 100644 --- a/chain/starknet/src/chain.rs +++ b/chain/starknet/src/chain.rs @@ -11,7 +11,7 @@ use graph::{ firehose_block_stream::FirehoseBlockStream, BasicBlockchainBuilder, Block, BlockIngestor, BlockPtr, Blockchain, BlockchainBuilder, BlockchainKind, EmptyNodeCapabilities, IngestorError, NoopDecoderHook, NoopRuntimeAdapter, - RuntimeAdapter as RuntimeAdapterTrait, + RuntimeAdapter as RuntimeAdapterTrait, TriggerFilterWrapper, }, cheap_clone::CheapClone, components::{ @@ -115,7 +115,7 @@ impl Blockchain for Chain { deployment: DeploymentLocator, store: impl DeploymentCursorTracker, start_blocks: Vec, - filter: Arc, + filter: Arc<&TriggerFilterWrapper>, unified_api_version: UnifiedMappingApiVersion, ) -> Result>, Error> { self.block_stream_builder @@ -125,7 +125,7 @@ impl Blockchain for Chain { store.firehose_cursor(), start_blocks, store.block_ptr(), - filter, + filter.filter.clone(), unified_api_version, ) .await diff --git a/chain/substreams/src/chain.rs b/chain/substreams/src/chain.rs index 28ef4bdc38b..b027edd3351 100644 --- a/chain/substreams/src/chain.rs +++ b/chain/substreams/src/chain.rs @@ -4,7 +4,7 @@ use anyhow::Error; use graph::blockchain::client::ChainClient; use graph::blockchain::{ BasicBlockchainBuilder, BlockIngestor, BlockTime, EmptyNodeCapabilities, NoopDecoderHook, - NoopRuntimeAdapter, + NoopRuntimeAdapter, TriggerFilterWrapper, }; use graph::components::adapter::ChainId; use graph::components::store::DeploymentCursorTracker; @@ -140,7 +140,7 @@ impl Blockchain for Chain { deployment: DeploymentLocator, store: impl DeploymentCursorTracker, _start_blocks: Vec, - filter: Arc, + filter: Arc<&TriggerFilterWrapper>, _unified_api_version: UnifiedMappingApiVersion, ) -> Result>, Error> { self.block_stream_builder @@ -150,7 +150,7 @@ impl Blockchain for Chain { deployment, store.firehose_cursor(), store.block_ptr(), - filter, + filter.filter.clone(), ) .await } diff --git a/core/src/subgraph/context/mod.rs b/core/src/subgraph/context/mod.rs index 6ffc5a5aa12..7b7686a04fb 100644 --- a/core/src/subgraph/context/mod.rs +++ b/core/src/subgraph/context/mod.rs @@ -6,7 +6,7 @@ use crate::polling_monitor::{ use anyhow::{self, Error}; use bytes::Bytes; use graph::{ - blockchain::{BlockTime, Blockchain}, + blockchain::{BlockTime, Blockchain, TriggerFilterWrapper}, components::{ store::{DeploymentId, SubgraphFork}, subgraph::{HostMetrics, MappingError, RuntimeHost as _, SharedProofOfIndexing}, @@ -73,7 +73,7 @@ where pub(crate) instance: SubgraphInstance, pub instances: SubgraphKeepAlive, pub offchain_monitor: OffchainMonitor, - pub filter: Option, + pub filter: Option>, pub(crate) trigger_processor: Box>, pub(crate) decoder: Box>, } diff --git a/core/src/subgraph/runner.rs b/core/src/subgraph/runner.rs index cd341ce2f99..28b82410276 100644 --- a/core/src/subgraph/runner.rs +++ b/core/src/subgraph/runner.rs @@ -7,7 +7,9 @@ use atomic_refcell::AtomicRefCell; use graph::blockchain::block_stream::{ BlockStreamError, BlockStreamEvent, BlockWithTriggers, FirehoseCursor, }; -use graph::blockchain::{Block, BlockTime, Blockchain, DataSource as _, TriggerFilter as _}; +use graph::blockchain::{ + Block, BlockTime, Blockchain, DataSource as _, TriggerFilter as _, TriggerFilterWrapper, +}; use graph::components::store::{EmptyStore, GetScope, ReadStore, StoredDynamicDataSource}; use graph::components::subgraph::InstanceDSTemplate; use graph::components::{ @@ -116,7 +118,7 @@ where self.inputs.static_filters || self.ctx.hosts_len() > ENV_VARS.static_filters_threshold } - fn build_filter(&self) -> C::TriggerFilter { + fn build_filter(&self) -> TriggerFilterWrapper { let current_ptr = self.inputs.store.block_ptr(); let static_filters = self.is_static_filters_enabled(); @@ -130,8 +132,11 @@ where // if static_filters is not enabled we just stick to the filter based on all the data sources. if !static_filters { - return C::TriggerFilter::from_data_sources( - self.ctx.onchain_data_sources().filter(end_block_filter), + return TriggerFilterWrapper::new( + C::TriggerFilter::from_data_sources( + self.ctx.onchain_data_sources().filter(end_block_filter), + ), + None, ); } @@ -158,11 +163,11 @@ where filter.extend_with_template(templates.iter().filter_map(|ds| ds.as_onchain()).cloned()); - filter + TriggerFilterWrapper::new(filter, None) } #[cfg(debug_assertions)] - pub fn build_filter_for_test(&self) -> C::TriggerFilter { + pub fn build_filter_for_test(&self) -> TriggerFilterWrapper { self.build_filter() } diff --git a/core/src/subgraph/stream.rs b/core/src/subgraph/stream.rs index c1d767e3fcf..b71d5e908ae 100644 --- a/core/src/subgraph/stream.rs +++ b/core/src/subgraph/stream.rs @@ -1,13 +1,13 @@ use crate::subgraph::inputs::IndexingInputs; use anyhow::bail; use graph::blockchain::block_stream::{BlockStream, BufferedBlockStream}; -use graph::blockchain::Blockchain; +use graph::blockchain::{Blockchain, TriggerFilterWrapper}; use graph::prelude::{CheapClone, Error, SubgraphInstanceMetrics}; use std::sync::Arc; pub async fn new_block_stream( inputs: &IndexingInputs, - filter: &C::TriggerFilter, + filter: &TriggerFilterWrapper, metrics: &SubgraphInstanceMetrics, ) -> Result>, Error> { let is_firehose = inputs.chain.chain_client().is_firehose(); @@ -18,7 +18,7 @@ pub async fn new_block_stream( inputs.deployment.clone(), inputs.store.cheap_clone(), inputs.start_blocks.clone(), - Arc::new(filter.clone()), + Arc::new(filter), inputs.unified_api_version.clone(), ) .await diff --git a/graph/src/blockchain/mock.rs b/graph/src/blockchain/mock.rs index c89eca95727..6b1eaba4ce7 100644 --- a/graph/src/blockchain/mock.rs +++ b/graph/src/blockchain/mock.rs @@ -14,10 +14,7 @@ use serde::Deserialize; use std::{collections::HashSet, convert::TryFrom, sync::Arc}; use super::{ - block_stream::{self, BlockStream, FirehoseCursor}, - client::ChainClient, - BlockIngestor, BlockTime, EmptyNodeCapabilities, HostFn, IngestorError, MappingTriggerTrait, - NoopDecoderHook, TriggerWithHandler, + block_stream::{self, BlockStream, FirehoseCursor}, client::ChainClient, BlockIngestor, BlockTime, EmptyNodeCapabilities, HostFn, IngestorError, MappingTriggerTrait, NoopDecoderHook, TriggerFilterWrapper, TriggerWithHandler }; use super::{ @@ -218,31 +215,37 @@ impl UnresolvedDataSourceTemplate for MockUnresolvedDataSource pub struct MockTriggersAdapter; #[async_trait] -impl TriggersAdapter for MockTriggersAdapter { +impl TriggersAdapter for MockTriggersAdapter { async fn ancestor_block( &self, _ptr: BlockPtr, _offset: BlockNumber, _root: Option, - ) -> Result, Error> { + ) -> Result, Error> { todo!() } async fn scan_triggers( &self, - _from: crate::components::store::BlockNumber, - _to: crate::components::store::BlockNumber, - _filter: &C::TriggerFilter, - ) -> Result<(Vec>, BlockNumber), Error> { - todo!() + from: crate::components::store::BlockNumber, + to: crate::components::store::BlockNumber, + filter: &MockTriggerFilter, + ) -> Result< + ( + Vec>, + BlockNumber, + ), + Error, + > { + blocks_with_triggers(from, to, filter).await } async fn triggers_in_block( &self, _logger: &slog::Logger, - _block: C::Block, - _filter: &C::TriggerFilter, - ) -> Result, Error> { + _block: MockBlock, + _filter: &MockTriggerFilter, + ) -> Result, Error> { todo!() } @@ -255,6 +258,26 @@ impl TriggersAdapter for MockTriggersAdapter { } } +async fn blocks_with_triggers( + _from: crate::components::store::BlockNumber, + to: crate::components::store::BlockNumber, + _filter: &MockTriggerFilter, +) -> Result< + ( + Vec>, + BlockNumber, + ), + Error, +> { + Ok(( + vec![BlockWithTriggers { + block: MockBlock { number: 0 }, + trigger_data: vec![MockTriggerData], + }], + to, + )) +} + #[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd)] pub struct MockTriggerData; @@ -347,7 +370,7 @@ impl Blockchain for MockBlockchain { _deployment: DeploymentLocator, _store: impl DeploymentCursorTracker, _start_blocks: Vec, - _filter: Arc, + _filter: Arc<&TriggerFilterWrapper>, _unified_api_version: UnifiedMappingApiVersion, ) -> Result>, Error> { todo!() diff --git a/graph/src/blockchain/mod.rs b/graph/src/blockchain/mod.rs index 1b897440b9b..62b93e61413 100644 --- a/graph/src/blockchain/mod.rs +++ b/graph/src/blockchain/mod.rs @@ -26,7 +26,7 @@ use crate::{ }, data::subgraph::{UnifiedMappingApiVersion, MIN_SPEC_VERSION}, data_source::{self, DataSourceTemplateInfo}, - prelude::DataSourceContext, + prelude::{DataSourceContext, DeploymentHash}, runtime::{gas::GasCounter, AscHeap, HostExportError}, }; use crate::{ @@ -189,7 +189,7 @@ pub trait Blockchain: Debug + Sized + Send + Sync + Unpin + 'static { deployment: DeploymentLocator, store: impl DeploymentCursorTracker, start_blocks: Vec, - filter: Arc, + filter: Arc<&TriggerFilterWrapper>, unified_api_version: UnifiedMappingApiVersion, ) -> Result>, Error>; @@ -247,6 +247,20 @@ impl From for IngestorError { } } +pub struct TriggerFilterWrapper { + pub filter: Arc, + _subgraph_filter: Option, +} + +impl TriggerFilterWrapper { + pub fn new(filter: C::TriggerFilter, subgraph_filter: Option) -> Self { + Self { + filter: Arc::new(filter), + _subgraph_filter: subgraph_filter, + } + } +} + pub trait TriggerFilter: Default + Clone + Send + Sync { fn from_data_sources<'a>( data_sources: impl Iterator + Clone, diff --git a/tests/tests/runner_tests.rs b/tests/tests/runner_tests.rs index 169836f2390..0ff9f40d537 100644 --- a/tests/tests/runner_tests.rs +++ b/tests/tests/runner_tests.rs @@ -500,10 +500,10 @@ async fn substreams_trigger_filter_construction() -> anyhow::Result<()> { let runner = ctx.runner_substreams(test_ptr(0)).await; let filter = runner.build_filter_for_test(); - assert_eq!(filter.module_name(), "graph_out"); - assert_eq!(filter.modules().as_ref().unwrap().modules.len(), 2); - assert_eq!(filter.start_block().unwrap(), 0); - assert_eq!(filter.data_sources_len(), 1); + assert_eq!(filter.filter.module_name(), "graph_out"); + assert_eq!(filter.filter.modules().as_ref().unwrap().modules.len(), 2); + assert_eq!(filter.filter.start_block().unwrap(), 0); + assert_eq!(filter.filter.data_sources_len(), 1); Ok(()) } @@ -525,7 +525,7 @@ async fn end_block() -> anyhow::Result<()> { let runner = ctx.runner(block_ptr.clone()).await; let runner = runner.run_for_test(false).await.unwrap(); let filter = runner.context().filter.as_ref().unwrap(); - let addresses = filter.log().contract_addresses().collect::>(); + let addresses = filter.filter.log().contract_addresses().collect::>(); if should_contain_addr { assert!(addresses.contains(&addr)); From 63a38bb38d5888866e706ce9fd01197c112dafa1 Mon Sep 17 00:00:00 2001 From: incrypto32 Date: Fri, 12 Jul 2024 16:21:18 +0530 Subject: [PATCH 03/29] graph,chain: add build_subgraph_block_stream method --- chain/ethereum/src/chain.rs | 26 ++++++++++++++++++++++++++ chain/near/src/chain.rs | 12 ++++++++++++ chain/starknet/src/chain.rs | 12 ++++++++++++ chain/substreams/src/block_stream.rs | 14 +++++++++++++- graph/src/blockchain/block_stream.rs | 12 +++++++++++- graph/src/blockchain/mod.rs | 4 ++-- tests/src/fixture/mod.rs | 26 +++++++++++++++++++++++++- 7 files changed, 101 insertions(+), 5 deletions(-) diff --git a/chain/ethereum/src/chain.rs b/chain/ethereum/src/chain.rs index 458265c9b87..fb2c5fafa59 100644 --- a/chain/ethereum/src/chain.rs +++ b/chain/ethereum/src/chain.rs @@ -122,6 +122,17 @@ impl BlockStreamBuilder for EthereumStreamBuilder { unimplemented!() } + async fn build_subgraph_block_stream( + &self, + _chain: &Chain, + _deployment: DeploymentLocator, + _start_blocks: Vec, + _subgraph_current_block: Option, + _filter: Arc<&TriggerFilterWrapper>, + _unified_api_version: UnifiedMappingApiVersion, + ) -> Result>> { + unimplemented!() + } async fn build_polling( &self, chain: &Chain, @@ -414,6 +425,21 @@ impl Blockchain for Chain { unified_api_version: UnifiedMappingApiVersion, ) -> Result>, Error> { let current_ptr = store.block_ptr(); + + if filter.subgraph_filter.is_some() { + return self + .block_stream_builder + .build_subgraph_block_stream( + self, + deployment, + start_blocks, + current_ptr, + filter, + unified_api_version, + ) + .await; + } + match self.chain_client().as_ref() { ChainClient::Rpc(_) => { self.block_stream_builder diff --git a/chain/near/src/chain.rs b/chain/near/src/chain.rs index 962978304a9..a5be2d260c3 100644 --- a/chain/near/src/chain.rs +++ b/chain/near/src/chain.rs @@ -109,6 +109,18 @@ impl BlockStreamBuilder for NearStreamBuilder { ))) } + async fn build_subgraph_block_stream( + &self, + _chain: &Chain, + _deployment: DeploymentLocator, + _start_blocks: Vec, + _subgraph_current_block: Option, + _filter: Arc<&TriggerFilterWrapper>, + _unified_api_version: UnifiedMappingApiVersion, + ) -> Result>> { + unimplemented!() + } + async fn build_firehose( &self, chain: &Chain, diff --git a/chain/starknet/src/chain.rs b/chain/starknet/src/chain.rs index 442474f507b..2419538435f 100644 --- a/chain/starknet/src/chain.rs +++ b/chain/starknet/src/chain.rs @@ -196,6 +196,18 @@ impl BlockStreamBuilder for StarknetStreamBuilder { unimplemented!() } + async fn build_subgraph_block_stream( + &self, + _chain: &Chain, + _deployment: DeploymentLocator, + _start_blocks: Vec, + _subgraph_current_block: Option, + _filter: Arc<&TriggerFilterWrapper>, + _unified_api_version: UnifiedMappingApiVersion, + ) -> Result>> { + unimplemented!() + } + async fn build_firehose( &self, chain: &Chain, diff --git a/chain/substreams/src/block_stream.rs b/chain/substreams/src/block_stream.rs index 8844df0610e..a2a233961d2 100644 --- a/chain/substreams/src/block_stream.rs +++ b/chain/substreams/src/block_stream.rs @@ -7,7 +7,7 @@ use graph::{ BlockStream, BlockStreamBuilder as BlockStreamBuilderTrait, FirehoseCursor, }, substreams_block_stream::SubstreamsBlockStream, - Blockchain, + Blockchain, TriggerFilterWrapper, }, components::store::DeploymentLocator, data::subgraph::UnifiedMappingApiVersion, @@ -99,6 +99,18 @@ impl BlockStreamBuilderTrait for BlockStreamBuilder { unimplemented!() } + async fn build_subgraph_block_stream( + &self, + _chain: &Chain, + _deployment: DeploymentLocator, + _start_blocks: Vec, + _subgraph_current_block: Option, + _filter: Arc<&TriggerFilterWrapper>, + _unified_api_version: UnifiedMappingApiVersion, + ) -> Result>> { + unimplemented!() + } + async fn build_polling( &self, _chain: &Chain, diff --git a/graph/src/blockchain/block_stream.rs b/graph/src/blockchain/block_stream.rs index 25a923dd502..3f792b3688e 100644 --- a/graph/src/blockchain/block_stream.rs +++ b/graph/src/blockchain/block_stream.rs @@ -12,7 +12,7 @@ use thiserror::Error; use tokio::sync::mpsc::{self, Receiver, Sender}; use super::substreams_block_stream::SubstreamsLogData; -use super::{Block, BlockPtr, BlockTime, Blockchain}; +use super::{Block, BlockPtr, BlockTime, Blockchain, TriggerFilterWrapper}; use crate::anyhow::Result; use crate::components::store::{BlockNumber, DeploymentLocator}; use crate::data::subgraph::UnifiedMappingApiVersion; @@ -148,6 +148,16 @@ pub trait BlockStreamBuilder: Send + Sync { filter: Arc, unified_api_version: UnifiedMappingApiVersion, ) -> Result>>; + + async fn build_subgraph_block_stream( + &self, + chain: &C, + deployment: DeploymentLocator, + start_blocks: Vec, + subgraph_current_block: Option, + filter: Arc<&TriggerFilterWrapper>, + unified_api_version: UnifiedMappingApiVersion, + ) -> Result>>; } #[derive(Debug, Clone)] diff --git a/graph/src/blockchain/mod.rs b/graph/src/blockchain/mod.rs index 62b93e61413..cbaff052706 100644 --- a/graph/src/blockchain/mod.rs +++ b/graph/src/blockchain/mod.rs @@ -249,14 +249,14 @@ impl From for IngestorError { pub struct TriggerFilterWrapper { pub filter: Arc, - _subgraph_filter: Option, + pub subgraph_filter: Option, } impl TriggerFilterWrapper { pub fn new(filter: C::TriggerFilter, subgraph_filter: Option) -> Self { Self { filter: Arc::new(filter), - _subgraph_filter: subgraph_filter, + subgraph_filter, } } } diff --git a/tests/src/fixture/mod.rs b/tests/src/fixture/mod.rs index ebed1d3a115..2ecd386a087 100644 --- a/tests/src/fixture/mod.rs +++ b/tests/src/fixture/mod.rs @@ -14,7 +14,7 @@ use graph::blockchain::block_stream::{ }; use graph::blockchain::{ Block, BlockHash, BlockPtr, Blockchain, BlockchainMap, ChainIdentifier, RuntimeAdapter, - TriggersAdapter, TriggersAdapterSelector, + TriggerFilterWrapper, TriggersAdapter, TriggersAdapterSelector, }; use graph::cheap_clone::CheapClone; use graph::components::adapter::ChainId; @@ -716,6 +716,18 @@ impl BlockStreamBuilder for MutexBlockStreamBuilder { unimplemented!(); } + async fn build_subgraph_block_stream( + &self, + _chain: &C, + _deployment: DeploymentLocator, + _start_blocks: Vec, + _subgraph_current_block: Option, + _filter: Arc<&TriggerFilterWrapper>, + _unified_api_version: graph::data::subgraph::UnifiedMappingApiVersion, + ) -> anyhow::Result>> { + unimplemented!() + } + async fn build_polling( &self, _chain: &C, @@ -755,6 +767,18 @@ where unimplemented!() } + async fn build_subgraph_block_stream( + &self, + _chain: &C, + _deployment: DeploymentLocator, + _start_blocks: Vec, + _subgraph_current_block: Option, + _filter: Arc<&TriggerFilterWrapper>, + _unified_api_version: graph::data::subgraph::UnifiedMappingApiVersion, + ) -> anyhow::Result>> { + unimplemented!() + } + async fn build_firehose( &self, _chain: &C, From 1c0fc742abf4f8207afd9b1258c7bc7354e80c98 Mon Sep 17 00:00:00 2001 From: incrypto32 Date: Mon, 15 Jul 2024 16:45:35 +0530 Subject: [PATCH 04/29] graph,core,chain: use TriggerFilterWrapper in PollingBlockStream --- chain/arweave/src/chain.rs | 2 +- chain/cosmos/src/chain.rs | 2 +- chain/ethereum/src/chain.rs | 77 +++++++++++++++++--- chain/near/src/chain.rs | 6 +- chain/starknet/src/chain.rs | 6 +- chain/substreams/src/block_stream.rs | 4 +- chain/substreams/src/chain.rs | 2 +- core/src/subgraph/runner.rs | 4 +- core/src/subgraph/stream.rs | 4 +- graph/src/blockchain/block_stream.rs | 4 +- graph/src/blockchain/mock.rs | 2 +- graph/src/blockchain/mod.rs | 18 ++++- graph/src/blockchain/polling_block_stream.rs | 10 +-- tests/src/fixture/mod.rs | 8 +- 14 files changed, 110 insertions(+), 39 deletions(-) diff --git a/chain/arweave/src/chain.rs b/chain/arweave/src/chain.rs index 70c03d832dd..9e6167b5678 100644 --- a/chain/arweave/src/chain.rs +++ b/chain/arweave/src/chain.rs @@ -119,7 +119,7 @@ impl Blockchain for Chain { deployment: DeploymentLocator, store: impl DeploymentCursorTracker, start_blocks: Vec, - filter: Arc<&TriggerFilterWrapper>, + filter: Arc>, unified_api_version: UnifiedMappingApiVersion, ) -> Result>, Error> { let adapter = self diff --git a/chain/cosmos/src/chain.rs b/chain/cosmos/src/chain.rs index b21acb3a8e6..6c88e710491 100644 --- a/chain/cosmos/src/chain.rs +++ b/chain/cosmos/src/chain.rs @@ -113,7 +113,7 @@ impl Blockchain for Chain { deployment: DeploymentLocator, store: impl DeploymentCursorTracker, start_blocks: Vec, - filter: Arc<&TriggerFilterWrapper>, + filter: Arc>, unified_api_version: UnifiedMappingApiVersion, ) -> Result>, Error> { let adapter = self diff --git a/chain/ethereum/src/chain.rs b/chain/ethereum/src/chain.rs index fb2c5fafa59..2f91a05e817 100644 --- a/chain/ethereum/src/chain.rs +++ b/chain/ethereum/src/chain.rs @@ -124,25 +124,80 @@ impl BlockStreamBuilder for EthereumStreamBuilder { async fn build_subgraph_block_stream( &self, - _chain: &Chain, - _deployment: DeploymentLocator, - _start_blocks: Vec, - _subgraph_current_block: Option, - _filter: Arc<&TriggerFilterWrapper>, - _unified_api_version: UnifiedMappingApiVersion, + chain: &Chain, + deployment: DeploymentLocator, + start_blocks: Vec, + subgraph_current_block: Option, + filter: Arc>, + unified_api_version: UnifiedMappingApiVersion, ) -> Result>> { - unimplemented!() + let requirements = filter.filter.node_capabilities(); + let adapter = chain + .triggers_adapter(&deployment, &requirements, unified_api_version.clone()) + .unwrap_or_else(|_| { + panic!( + "no adapter for network {} with capabilities {}", + chain.name, requirements + ) + }); + + let logger = chain + .logger_factory + .subgraph_logger(&deployment) + .new(o!("component" => "BlockStream")); + let chain_store = chain.chain_store(); + let chain_head_update_stream = chain + .chain_head_update_listener + .subscribe(chain.name.to_string(), logger.clone()); + + // Special case: Detect Celo and set the threshold to 0, so that eth_getLogs is always used. + // This is ok because Celo blocks are always final. And we _need_ to do this because + // some events appear only in eth_getLogs but not in transaction receipts. + // See also ca0edc58-0ec5-4c89-a7dd-2241797f5e50. + let chain_id = match chain.chain_client().as_ref() { + ChainClient::Rpc(adapter) => { + adapter + .cheapest() + .await + .ok_or(anyhow!("unable to get eth adapter for chan_id call"))? + .chain_id() + .await? + } + _ => panic!("expected rpc when using polling blockstream"), + }; + let reorg_threshold = match CELO_CHAIN_IDS.contains(&chain_id) { + false => chain.reorg_threshold, + true => 0, + }; + + Ok(Box::new(PollingBlockStream::new( + chain_store, + chain_head_update_stream, + adapter, + chain.node_id.clone(), + deployment.hash, + filter, + start_blocks, + reorg_threshold, + logger, + ENV_VARS.max_block_range_size, + ENV_VARS.target_triggers_per_block_range, + unified_api_version, + subgraph_current_block, + ))) } + async fn build_polling( &self, chain: &Chain, deployment: DeploymentLocator, start_blocks: Vec, subgraph_current_block: Option, - filter: Arc<::TriggerFilter>, + filter: Arc>, unified_api_version: UnifiedMappingApiVersion, ) -> Result>> { - let requirements = filter.node_capabilities(); + + let requirements = filter.filter.node_capabilities(); let adapter = chain .triggers_adapter(&deployment, &requirements, unified_api_version.clone()) .unwrap_or_else(|_| { @@ -421,7 +476,7 @@ impl Blockchain for Chain { deployment: DeploymentLocator, store: impl DeploymentCursorTracker, start_blocks: Vec, - filter: Arc<&TriggerFilterWrapper>, + filter: Arc>, unified_api_version: UnifiedMappingApiVersion, ) -> Result>, Error> { let current_ptr = store.block_ptr(); @@ -448,7 +503,7 @@ impl Blockchain for Chain { deployment, start_blocks, current_ptr, - filter.filter.clone(), + filter, unified_api_version, ) .await diff --git a/chain/near/src/chain.rs b/chain/near/src/chain.rs index a5be2d260c3..86a77c2fc61 100644 --- a/chain/near/src/chain.rs +++ b/chain/near/src/chain.rs @@ -115,7 +115,7 @@ impl BlockStreamBuilder for NearStreamBuilder { _deployment: DeploymentLocator, _start_blocks: Vec, _subgraph_current_block: Option, - _filter: Arc<&TriggerFilterWrapper>, + _filter: Arc>, _unified_api_version: UnifiedMappingApiVersion, ) -> Result>> { unimplemented!() @@ -164,7 +164,7 @@ impl BlockStreamBuilder for NearStreamBuilder { _deployment: DeploymentLocator, _start_blocks: Vec, _subgraph_current_block: Option, - _filter: Arc<::TriggerFilter>, + _filter: Arc>, _unified_api_version: UnifiedMappingApiVersion, ) -> Result>> { todo!() @@ -242,7 +242,7 @@ impl Blockchain for Chain { deployment: DeploymentLocator, store: impl DeploymentCursorTracker, start_blocks: Vec, - filter: Arc<&TriggerFilterWrapper>, + filter: Arc>, unified_api_version: UnifiedMappingApiVersion, ) -> Result>, Error> { if self.prefer_substreams { diff --git a/chain/starknet/src/chain.rs b/chain/starknet/src/chain.rs index 2419538435f..d926b31877b 100644 --- a/chain/starknet/src/chain.rs +++ b/chain/starknet/src/chain.rs @@ -115,7 +115,7 @@ impl Blockchain for Chain { deployment: DeploymentLocator, store: impl DeploymentCursorTracker, start_blocks: Vec, - filter: Arc<&TriggerFilterWrapper>, + filter: Arc>, unified_api_version: UnifiedMappingApiVersion, ) -> Result>, Error> { self.block_stream_builder @@ -202,7 +202,7 @@ impl BlockStreamBuilder for StarknetStreamBuilder { _deployment: DeploymentLocator, _start_blocks: Vec, _subgraph_current_block: Option, - _filter: Arc<&TriggerFilterWrapper>, + _filter: Arc>, _unified_api_version: UnifiedMappingApiVersion, ) -> Result>> { unimplemented!() @@ -251,7 +251,7 @@ impl BlockStreamBuilder for StarknetStreamBuilder { _deployment: DeploymentLocator, _start_blocks: Vec, _subgraph_current_block: Option, - _filter: Arc, + _filter: Arc>, _unified_api_version: UnifiedMappingApiVersion, ) -> Result>> { panic!("StarkNet does not support polling block stream") diff --git a/chain/substreams/src/block_stream.rs b/chain/substreams/src/block_stream.rs index a2a233961d2..2d6eb902409 100644 --- a/chain/substreams/src/block_stream.rs +++ b/chain/substreams/src/block_stream.rs @@ -105,7 +105,7 @@ impl BlockStreamBuilderTrait for BlockStreamBuilder { _deployment: DeploymentLocator, _start_blocks: Vec, _subgraph_current_block: Option, - _filter: Arc<&TriggerFilterWrapper>, + _filter: Arc>, _unified_api_version: UnifiedMappingApiVersion, ) -> Result>> { unimplemented!() @@ -117,7 +117,7 @@ impl BlockStreamBuilderTrait for BlockStreamBuilder { _deployment: DeploymentLocator, _start_blocks: Vec, _subgraph_current_block: Option, - _filter: Arc, + _filter: Arc>, _unified_api_version: UnifiedMappingApiVersion, ) -> Result>> { unimplemented!("polling block stream is not support for substreams") diff --git a/chain/substreams/src/chain.rs b/chain/substreams/src/chain.rs index b027edd3351..044617603d8 100644 --- a/chain/substreams/src/chain.rs +++ b/chain/substreams/src/chain.rs @@ -140,7 +140,7 @@ impl Blockchain for Chain { deployment: DeploymentLocator, store: impl DeploymentCursorTracker, _start_blocks: Vec, - filter: Arc<&TriggerFilterWrapper>, + filter: Arc>, _unified_api_version: UnifiedMappingApiVersion, ) -> Result>, Error> { self.block_stream_builder diff --git a/core/src/subgraph/runner.rs b/core/src/subgraph/runner.rs index 28b82410276..64f60e2c378 100644 --- a/core/src/subgraph/runner.rs +++ b/core/src/subgraph/runner.rs @@ -167,7 +167,7 @@ where } #[cfg(debug_assertions)] - pub fn build_filter_for_test(&self) -> TriggerFilterWrapper { + pub fn build_filter_for_test(&self) -> TriggerFilterWrapper { self.build_filter() } @@ -214,7 +214,7 @@ where let mut block_stream = new_block_stream( &self.inputs, - self.ctx.filter.as_ref().unwrap(), // Safe to unwrap as we just called `build_filter` in the previous line + self.ctx.filter.cheap_clone().unwrap(), // Safe to unwrap as we just called `build_filter` in the previous line &self.metrics.subgraph, ) .await? diff --git a/core/src/subgraph/stream.rs b/core/src/subgraph/stream.rs index b71d5e908ae..cfd41808e27 100644 --- a/core/src/subgraph/stream.rs +++ b/core/src/subgraph/stream.rs @@ -7,7 +7,7 @@ use std::sync::Arc; pub async fn new_block_stream( inputs: &IndexingInputs, - filter: &TriggerFilterWrapper, + filter: TriggerFilterWrapper, metrics: &SubgraphInstanceMetrics, ) -> Result>, Error> { let is_firehose = inputs.chain.chain_client().is_firehose(); @@ -18,7 +18,7 @@ pub async fn new_block_stream( inputs.deployment.clone(), inputs.store.cheap_clone(), inputs.start_blocks.clone(), - Arc::new(filter), + Arc::new(filter.clone()), inputs.unified_api_version.clone(), ) .await diff --git a/graph/src/blockchain/block_stream.rs b/graph/src/blockchain/block_stream.rs index 3f792b3688e..2b4deb0e15b 100644 --- a/graph/src/blockchain/block_stream.rs +++ b/graph/src/blockchain/block_stream.rs @@ -145,7 +145,7 @@ pub trait BlockStreamBuilder: Send + Sync { deployment: DeploymentLocator, start_blocks: Vec, subgraph_current_block: Option, - filter: Arc, + filter: Arc>, unified_api_version: UnifiedMappingApiVersion, ) -> Result>>; @@ -155,7 +155,7 @@ pub trait BlockStreamBuilder: Send + Sync { deployment: DeploymentLocator, start_blocks: Vec, subgraph_current_block: Option, - filter: Arc<&TriggerFilterWrapper>, + filter: Arc>, unified_api_version: UnifiedMappingApiVersion, ) -> Result>>; } diff --git a/graph/src/blockchain/mock.rs b/graph/src/blockchain/mock.rs index 6b1eaba4ce7..5321547f5dd 100644 --- a/graph/src/blockchain/mock.rs +++ b/graph/src/blockchain/mock.rs @@ -370,7 +370,7 @@ impl Blockchain for MockBlockchain { _deployment: DeploymentLocator, _store: impl DeploymentCursorTracker, _start_blocks: Vec, - _filter: Arc<&TriggerFilterWrapper>, + _filter: Arc>, _unified_api_version: UnifiedMappingApiVersion, ) -> Result>, Error> { todo!() diff --git a/graph/src/blockchain/mod.rs b/graph/src/blockchain/mod.rs index cbaff052706..39a42f3ff48 100644 --- a/graph/src/blockchain/mod.rs +++ b/graph/src/blockchain/mod.rs @@ -189,7 +189,7 @@ pub trait Blockchain: Debug + Sized + Send + Sync + Unpin + 'static { deployment: DeploymentLocator, store: impl DeploymentCursorTracker, start_blocks: Vec, - filter: Arc<&TriggerFilterWrapper>, + filter: Arc>, unified_api_version: UnifiedMappingApiVersion, ) -> Result>, Error>; @@ -247,6 +247,7 @@ impl From for IngestorError { } } +#[derive(Debug)] pub struct TriggerFilterWrapper { pub filter: Arc, pub subgraph_filter: Option, @@ -261,6 +262,21 @@ impl TriggerFilterWrapper { } } +impl Clone for TriggerFilterWrapper { + fn clone(&self) -> Self { + Self { + filter: self.filter.cheap_clone(), + subgraph_filter: self.subgraph_filter.cheap_clone(), + } + } +} + +impl CheapClone for TriggerFilterWrapper { + fn cheap_clone(&self) -> Self { + self.clone() + } +} + pub trait TriggerFilter: Default + Clone + Send + Sync { fn from_data_sources<'a>( data_sources: impl Iterator + Clone, diff --git a/graph/src/blockchain/polling_block_stream.rs b/graph/src/blockchain/polling_block_stream.rs index ce3fdf2a4ef..ed4416fda70 100644 --- a/graph/src/blockchain/polling_block_stream.rs +++ b/graph/src/blockchain/polling_block_stream.rs @@ -11,7 +11,7 @@ use super::block_stream::{ BlockStream, BlockStreamError, BlockStreamEvent, BlockWithTriggers, ChainHeadUpdateStream, FirehoseCursor, TriggersAdapter, BUFFERED_BLOCK_STREAM_SIZE, }; -use super::{Block, BlockPtr, Blockchain}; +use super::{Block, BlockPtr, Blockchain, TriggerFilterWrapper}; use crate::components::store::BlockNumber; use crate::data::subgraph::UnifiedMappingApiVersion; @@ -85,7 +85,7 @@ where // This is not really a block number, but the (unsigned) difference // between two block numbers reorg_threshold: BlockNumber, - filter: Arc, + filter: Arc>, start_blocks: Vec, logger: Logger, previous_triggers_per_block: f64, @@ -149,7 +149,7 @@ where adapter: Arc>, node_id: NodeId, subgraph_id: DeploymentHash, - filter: Arc, + filter: Arc>, start_blocks: Vec, reorg_threshold: BlockNumber, logger: Logger, @@ -379,7 +379,7 @@ where ); // Update with actually scanned range, to account for any skipped null blocks. - let (blocks, to) = self.adapter.scan_triggers(from, to, &self.filter).await?; + let (blocks, to) = self.adapter.scan_triggers(from, to, &self.filter.filter.clone()).await?; let range_size = to - from + 1; // If the target block (`to`) is within the reorg threshold, indicating no non-null finalized blocks are @@ -469,7 +469,7 @@ where // Note that head_ancestor is a child of subgraph_ptr. let block = self .adapter - .triggers_in_block(&self.logger, head_ancestor, &self.filter) + .triggers_in_block(&self.logger, head_ancestor, &self.filter.filter.clone()) .await?; Ok(ReconciliationStep::ProcessDescendantBlocks(vec![block], 1)) } else { diff --git a/tests/src/fixture/mod.rs b/tests/src/fixture/mod.rs index 2ecd386a087..a6dbd650a3e 100644 --- a/tests/src/fixture/mod.rs +++ b/tests/src/fixture/mod.rs @@ -722,7 +722,7 @@ impl BlockStreamBuilder for MutexBlockStreamBuilder { _deployment: DeploymentLocator, _start_blocks: Vec, _subgraph_current_block: Option, - _filter: Arc<&TriggerFilterWrapper>, + _filter: Arc>, _unified_api_version: graph::data::subgraph::UnifiedMappingApiVersion, ) -> anyhow::Result>> { unimplemented!() @@ -734,7 +734,7 @@ impl BlockStreamBuilder for MutexBlockStreamBuilder { _deployment: DeploymentLocator, _start_blocks: Vec, _subgraph_current_block: Option, - _filter: Arc<::TriggerFilter>, + _filter: Arc>, _unified_api_version: graph::data::subgraph::UnifiedMappingApiVersion, ) -> anyhow::Result>> { unimplemented!("only firehose mode should be used for tests") @@ -773,7 +773,7 @@ where _deployment: DeploymentLocator, _start_blocks: Vec, _subgraph_current_block: Option, - _filter: Arc<&TriggerFilterWrapper>, + _filter: Arc>, _unified_api_version: graph::data::subgraph::UnifiedMappingApiVersion, ) -> anyhow::Result>> { unimplemented!() @@ -808,7 +808,7 @@ where _deployment: DeploymentLocator, _start_blocks: Vec, _subgraph_current_block: Option, - _filter: Arc, + _filter: Arc>, _unified_api_version: graph::data::subgraph::UnifiedMappingApiVersion, ) -> anyhow::Result>> { unimplemented!("only firehose mode should be used for tests") From 6ec034f242a779cfadd5ded651fbeb81b1b8ef50 Mon Sep 17 00:00:00 2001 From: incrypto32 Date: Mon, 15 Jul 2024 17:44:21 +0530 Subject: [PATCH 05/29] graph: created TriggersAdapterWrapper --- chain/ethereum/src/chain.rs | 4 ++- graph/src/blockchain/block_stream.rs | 48 ++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/chain/ethereum/src/chain.rs b/chain/ethereum/src/chain.rs index 2f91a05e817..46a9bc3184e 100644 --- a/chain/ethereum/src/chain.rs +++ b/chain/ethereum/src/chain.rs @@ -62,6 +62,7 @@ use crate::{BufferedCallCache, NodeCapabilities}; use crate::{EthereumAdapter, RuntimeAdapter}; use graph::blockchain::block_stream::{ BlockStream, BlockStreamBuilder, BlockStreamError, BlockStreamMapper, FirehoseCursor, + TriggersAdaterWrapper, }; /// Celo Mainnet: 42220, Testnet Alfajores: 44787, Testnet Baklava: 62320 @@ -141,6 +142,8 @@ impl BlockStreamBuilder for EthereumStreamBuilder { ) }); + let adapter = Arc::new(TriggersAdaterWrapper::new(adapter)); + let logger = chain .logger_factory .subgraph_logger(&deployment) @@ -196,7 +199,6 @@ impl BlockStreamBuilder for EthereumStreamBuilder { filter: Arc>, unified_api_version: UnifiedMappingApiVersion, ) -> Result>> { - let requirements = filter.filter.node_capabilities(); let adapter = chain .triggers_adapter(&deployment, &requirements, unified_api_version.clone()) diff --git a/graph/src/blockchain/block_stream.rs b/graph/src/blockchain/block_stream.rs index 2b4deb0e15b..758209c343e 100644 --- a/graph/src/blockchain/block_stream.rs +++ b/graph/src/blockchain/block_stream.rs @@ -268,6 +268,54 @@ impl BlockWithTriggers { } } +pub struct TriggersAdaterWrapper { + pub adapter: Arc>, +} + +impl TriggersAdaterWrapper { + pub fn new(adapter: Arc>) -> Self { + Self { adapter } + } +} + +#[async_trait] +impl TriggersAdapter for TriggersAdaterWrapper { + async fn ancestor_block( + &self, + ptr: BlockPtr, + offset: BlockNumber, + root: Option, + ) -> Result, Error> { + self.adapter.ancestor_block(ptr, offset, root).await + } + + async fn scan_triggers( + &self, + from: BlockNumber, + to: BlockNumber, + filter: &C::TriggerFilter, + ) -> Result<(Vec>, BlockNumber), Error> { + self.adapter.scan_triggers(from, to, filter).await + } + + async fn triggers_in_block( + &self, + logger: &Logger, + block: C::Block, + filter: &C::TriggerFilter, + ) -> Result, Error> { + self.adapter.triggers_in_block(logger, block, filter).await + } + + async fn is_on_main_chain(&self, ptr: BlockPtr) -> Result { + self.adapter.is_on_main_chain(ptr).await + } + + async fn parent_ptr(&self, block: &BlockPtr) -> Result, Error> { + self.adapter.parent_ptr(block).await + } +} + #[async_trait] pub trait TriggersAdapter: Send + Sync { // Return the block that is `offset` blocks before the block pointed to by `ptr` from the local From 526893db87bd236d7c68c003be0d10b44c3b2921 Mon Sep 17 00:00:00 2001 From: incrypto32 Date: Tue, 16 Jul 2024 14:14:37 +0530 Subject: [PATCH 06/29] graph,core,chain: Add a wrapper enum for Triggers to handle subgraph datasource triggers --- chain/cosmos/src/chain.rs | 14 ++-- chain/ethereum/src/ethereum_adapter.rs | 6 +- chain/ethereum/src/tests.rs | 22 +++--- chain/near/src/chain.rs | 18 ++--- core/src/subgraph/context/instance/mod.rs | 1 + core/src/subgraph/runner.rs | 13 +++- graph/src/blockchain/block_stream.rs | 19 +++++- graph/src/blockchain/mock.rs | 7 +- graph/src/blockchain/mod.rs | 72 +++++++++++++++++++- graph/src/blockchain/polling_block_stream.rs | 13 +++- graph/src/data_source/mod.rs | 6 +- graph/src/env/mod.rs | 2 +- tests/src/fixture/ethereum.rs | 30 +++++--- 13 files changed, 176 insertions(+), 47 deletions(-) diff --git a/chain/cosmos/src/chain.rs b/chain/cosmos/src/chain.rs index 6c88e710491..868617968df 100644 --- a/chain/cosmos/src/chain.rs +++ b/chain/cosmos/src/chain.rs @@ -470,9 +470,12 @@ impl FirehoseMapperTrait for FirehoseMapper { #[cfg(test)] mod test { - use graph::prelude::{ - slog::{o, Discard, Logger}, - tokio, + use graph::{ + blockchain::Trigger, + prelude::{ + slog::{o, Discard, Logger}, + tokio, + }, }; use super::*; @@ -603,7 +606,10 @@ mod test { // they may not be in the same order for trigger in expected_triggers { assert!( - triggers.trigger_data.contains(&trigger), + triggers.trigger_data.iter().any(|t| match t { + Trigger::Chain(t) => t == &trigger, + _ => false, + }), "Expected trigger list to contain {:?}, but it only contains: {:?}", trigger, triggers.trigger_data diff --git a/chain/ethereum/src/ethereum_adapter.rs b/chain/ethereum/src/ethereum_adapter.rs index c4ea6323c7d..123f79bb4a8 100644 --- a/chain/ethereum/src/ethereum_adapter.rs +++ b/chain/ethereum/src/ethereum_adapter.rs @@ -2077,8 +2077,8 @@ async fn filter_call_triggers_from_unsuccessful_transactions( let transaction_hashes: BTreeSet = block .trigger_data .iter() - .filter_map(|trigger| match trigger { - EthereumTrigger::Call(call_trigger) => Some(call_trigger.transaction_hash), + .filter_map(|trigger| match trigger.as_chain() { + Some(EthereumTrigger::Call(call_trigger)) => Some(call_trigger.transaction_hash), _ => None, }) .collect::>>() @@ -2169,7 +2169,7 @@ async fn filter_call_triggers_from_unsuccessful_transactions( // Filter call triggers from unsuccessful transactions block.trigger_data.retain(|trigger| { - if let EthereumTrigger::Call(call_trigger) = trigger { + if let Some(EthereumTrigger::Call(call_trigger)) = trigger.as_chain() { // Unwrap: We already checked that those values exist transaction_success[&call_trigger.transaction_hash.unwrap()] } else { diff --git a/chain/ethereum/src/tests.rs b/chain/ethereum/src/tests.rs index 455a7c07432..00873f8ea87 100644 --- a/chain/ethereum/src/tests.rs +++ b/chain/ethereum/src/tests.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use graph::{ - blockchain::{block_stream::BlockWithTriggers, BlockPtr}, + blockchain::{block_stream::BlockWithTriggers, BlockPtr, Trigger}, prelude::{ web3::types::{Address, Bytes, Log, H160, H256, U64}, EthereumCall, LightEthereumBlock, @@ -107,10 +107,12 @@ fn test_trigger_ordering() { &logger, ); - assert_eq!( - block_with_triggers.trigger_data, - vec![log1, log2, call1, log3, call2, call4, call3, block2, block1] - ); + let expected = vec![log1, log2, call1, log3, call2, call4, call3, block2, block1] + .into_iter() + .map(|t| Trigger::Chain(t)) + .collect::>(); + + assert_eq!(block_with_triggers.trigger_data, expected); } #[test] @@ -203,8 +205,10 @@ fn test_trigger_dedup() { &logger, ); - assert_eq!( - block_with_triggers.trigger_data, - vec![log1, log2, call1, log3, call2, call3, block2, block1] - ); + let expected = vec![log1, log2, call1, log3, call2, call3, block2, block1] + .into_iter() + .map(|t| Trigger::Chain(t)) + .collect::>(); + + assert_eq!(block_with_triggers.trigger_data, expected); } diff --git a/chain/near/src/chain.rs b/chain/near/src/chain.rs index 86a77c2fc61..3211870f069 100644 --- a/chain/near/src/chain.rs +++ b/chain/near/src/chain.rs @@ -4,7 +4,7 @@ use graph::blockchain::firehose_block_ingestor::FirehoseBlockIngestor; use graph::blockchain::substreams_block_stream::SubstreamsBlockStream; use graph::blockchain::{ BasicBlockchainBuilder, BlockIngestor, BlockchainBuilder, BlockchainKind, NoopDecoderHook, - NoopRuntimeAdapter, TriggerFilterWrapper, + NoopRuntimeAdapter, Trigger, TriggerFilterWrapper, }; use graph::cheap_clone::CheapClone; use graph::components::adapter::ChainId; @@ -474,11 +474,13 @@ impl BlockStreamMapper for FirehoseMapper { .into_iter() .zip(receipt.into_iter()) .map(|(outcome, receipt)| { - NearTrigger::Receipt(Arc::new(trigger::ReceiptWithOutcome { - outcome, - receipt, - block: arc_block.clone(), - })) + Trigger::Chain(NearTrigger::Receipt(Arc::new( + trigger::ReceiptWithOutcome { + outcome, + receipt, + block: arc_block.clone(), + }, + ))) }) .collect(); @@ -985,8 +987,8 @@ mod test { .trigger_data .clone() .into_iter() - .filter_map(|x| match x { - crate::trigger::NearTrigger::Block(b) => b.header.clone().map(|x| x.height), + .filter_map(|x| match x.as_chain() { + Some(crate::trigger::NearTrigger::Block(b)) => b.header.clone().map(|x| x.height), _ => None, }) .collect() diff --git a/core/src/subgraph/context/instance/mod.rs b/core/src/subgraph/context/instance/mod.rs index 5a805f34095..fa723d3f18e 100644 --- a/core/src/subgraph/context/instance/mod.rs +++ b/core/src/subgraph/context/instance/mod.rs @@ -229,6 +229,7 @@ where TriggerData::Offchain(trigger) => self .offchain_hosts .matches_by_address(trigger.source.address().as_ref().map(|a| a.as_slice())), + TriggerData::Subgraph(_) => todo!(), // TODO(krishna) } } diff --git a/core/src/subgraph/runner.rs b/core/src/subgraph/runner.rs index 64f60e2c378..1060fb2f93c 100644 --- a/core/src/subgraph/runner.rs +++ b/core/src/subgraph/runner.rs @@ -8,7 +8,8 @@ use graph::blockchain::block_stream::{ BlockStreamError, BlockStreamEvent, BlockWithTriggers, FirehoseCursor, }; use graph::blockchain::{ - Block, BlockTime, Blockchain, DataSource as _, TriggerFilter as _, TriggerFilterWrapper, + Block, BlockTime, Blockchain, DataSource as _, Trigger, TriggerFilter as _, + TriggerFilterWrapper, }; use graph::components::store::{EmptyStore, GetScope, ReadStore, StoredDynamicDataSource}; use graph::components::subgraph::InstanceDSTemplate; @@ -328,7 +329,10 @@ where .match_and_decode_many( &logger, &block, - triggers.into_iter().map(TriggerData::Onchain), + triggers.into_iter().map(|t| match t { + Trigger::Chain(t) => TriggerData::Onchain(t), + Trigger::Subgraph(_) => todo!(), //TODO(krishna), + }), hosts_filter, &self.metrics.subgraph, ) @@ -487,7 +491,10 @@ where .match_and_decode_many( &logger, &block, - triggers.into_iter().map(TriggerData::Onchain), + triggers.into_iter().map(|t| match t { + Trigger::Chain(t) => TriggerData::Onchain(t), + Trigger::Subgraph(_) => todo!(), //TODO(krishna), + }), |_| Box::new(runtime_hosts.iter().map(Arc::as_ref)), &self.metrics.subgraph, ) diff --git a/graph/src/blockchain/block_stream.rs b/graph/src/blockchain/block_stream.rs index 758209c343e..93693bb0384 100644 --- a/graph/src/blockchain/block_stream.rs +++ b/graph/src/blockchain/block_stream.rs @@ -12,7 +12,7 @@ use thiserror::Error; use tokio::sync::mpsc::{self, Receiver, Sender}; use super::substreams_block_stream::SubstreamsLogData; -use super::{Block, BlockPtr, BlockTime, Blockchain, TriggerFilterWrapper}; +use super::{Block, BlockPtr, BlockTime, Blockchain, Trigger, TriggerFilterWrapper}; use crate::anyhow::Result; use crate::components::store::{BlockNumber, DeploymentLocator}; use crate::data::subgraph::UnifiedMappingApiVersion; @@ -208,7 +208,7 @@ impl AsRef> for FirehoseCursor { #[derive(Debug)] pub struct BlockWithTriggers { pub block: C::Block, - pub trigger_data: Vec, + pub trigger_data: Vec>, } impl Clone for BlockWithTriggers @@ -226,7 +226,15 @@ where impl BlockWithTriggers { /// Creates a BlockWithTriggers structure, which holds /// the trigger data ordered and without any duplicates. - pub fn new(block: C::Block, mut trigger_data: Vec, logger: &Logger) -> Self { + pub fn new(block: C::Block, trigger_data: Vec, logger: &Logger) -> Self { + let mut trigger_data = trigger_data + .into_iter() + .map(|trigger_data| { + let trigger = Trigger::Chain(trigger_data); + trigger + }) + .collect::>(); + // This is where triggers get sorted. trigger_data.sort(); @@ -266,6 +274,11 @@ impl BlockWithTriggers { pub fn parent_ptr(&self) -> Option { self.block.parent_ptr() } + + pub fn extend_triggers(&mut self, triggers: Vec>) { + self.trigger_data.extend(triggers); + self.trigger_data.sort(); + } } pub struct TriggersAdaterWrapper { diff --git a/graph/src/blockchain/mock.rs b/graph/src/blockchain/mock.rs index 5321547f5dd..e13a21878e4 100644 --- a/graph/src/blockchain/mock.rs +++ b/graph/src/blockchain/mock.rs @@ -14,7 +14,10 @@ use serde::Deserialize; use std::{collections::HashSet, convert::TryFrom, sync::Arc}; use super::{ - block_stream::{self, BlockStream, FirehoseCursor}, client::ChainClient, BlockIngestor, BlockTime, EmptyNodeCapabilities, HostFn, IngestorError, MappingTriggerTrait, NoopDecoderHook, TriggerFilterWrapper, TriggerWithHandler + block_stream::{self, BlockStream, FirehoseCursor}, + client::ChainClient, + BlockIngestor, BlockTime, EmptyNodeCapabilities, HostFn, IngestorError, MappingTriggerTrait, + NoopDecoderHook, Trigger, TriggerFilterWrapper, TriggerWithHandler, }; use super::{ @@ -272,7 +275,7 @@ async fn blocks_with_triggers( Ok(( vec![BlockWithTriggers { block: MockBlock { number: 0 }, - trigger_data: vec![MockTriggerData], + trigger_data: vec![Trigger::Chain(MockTriggerData)], }], to, )) diff --git a/graph/src/blockchain/mod.rs b/graph/src/blockchain/mod.rs index 39a42f3ff48..eccb0336b8e 100644 --- a/graph/src/blockchain/mod.rs +++ b/graph/src/blockchain/mod.rs @@ -25,7 +25,7 @@ use crate::{ trigger_processor::RunnableTriggers, }, data::subgraph::{UnifiedMappingApiVersion, MIN_SPEC_VERSION}, - data_source::{self, DataSourceTemplateInfo}, + data_source::{self, subgraph, DataSourceTemplateInfo}, prelude::{DataSourceContext, DeploymentHash}, runtime::{gas::GasCounter, AscHeap, HostExportError}, }; @@ -400,6 +400,76 @@ pub trait UnresolvedDataSource: ) -> Result; } +#[derive(Debug)] +pub enum Trigger { + Chain(C::TriggerData), + Subgraph(subgraph::TriggerData), +} + +impl Trigger { + pub fn as_chain(&self) -> Option<&C::TriggerData> { + match self { + Trigger::Chain(data) => Some(data), + _ => None, + } + } + + pub fn as_subgraph(&self) -> Option<&subgraph::TriggerData> { + match self { + Trigger::Subgraph(data) => Some(data), + _ => None, + } + } +} + +impl Eq for Trigger where C::TriggerData: Eq {} + +impl PartialEq for Trigger +where + C::TriggerData: PartialEq, +{ + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (Trigger::Chain(data1), Trigger::Chain(data2)) => data1 == data2, + (Trigger::Subgraph(a), Trigger::Subgraph(b)) => a == b, + _ => false, + } + } +} + +impl Clone for Trigger +where + C::TriggerData: Clone, +{ + fn clone(&self) -> Self { + match self { + Trigger::Chain(data) => Trigger::Chain(data.clone()), + Trigger::Subgraph(data) => Trigger::Subgraph(data.clone()), + } + } +} + +// TODO(krishna): Proper ordering for triggers +impl Ord for Trigger +where + C::TriggerData: Ord, +{ + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + match (self, other) { + (Trigger::Chain(data1), Trigger::Chain(data2)) => data1.cmp(data2), + (Trigger::Subgraph(_), Trigger::Chain(_)) => std::cmp::Ordering::Greater, + (Trigger::Chain(_), Trigger::Subgraph(_)) => std::cmp::Ordering::Less, + (Trigger::Subgraph(_), Trigger::Subgraph(_)) => std::cmp::Ordering::Equal, + } + } +} + +impl PartialOrd for Trigger { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + pub trait TriggerData { /// If there is an error when processing this trigger, this will called to add relevant context. /// For example an useful return is: `"block # (), transaction ". diff --git a/graph/src/blockchain/polling_block_stream.rs b/graph/src/blockchain/polling_block_stream.rs index ed4416fda70..64a84ff3b94 100644 --- a/graph/src/blockchain/polling_block_stream.rs +++ b/graph/src/blockchain/polling_block_stream.rs @@ -149,7 +149,7 @@ where adapter: Arc>, node_id: NodeId, subgraph_id: DeploymentHash, - filter: Arc>, + filter: Arc>, start_blocks: Vec, reorg_threshold: BlockNumber, logger: Logger, @@ -379,7 +379,10 @@ where ); // Update with actually scanned range, to account for any skipped null blocks. - let (blocks, to) = self.adapter.scan_triggers(from, to, &self.filter.filter.clone()).await?; + let (blocks, to) = self + .adapter + .scan_triggers(from, to, &self.filter.filter.clone()) + .await?; let range_size = to - from + 1; // If the target block (`to`) is within the reorg threshold, indicating no non-null finalized blocks are @@ -469,7 +472,11 @@ where // Note that head_ancestor is a child of subgraph_ptr. let block = self .adapter - .triggers_in_block(&self.logger, head_ancestor, &self.filter.filter.clone()) + .triggers_in_block( + &self.logger, + head_ancestor, + &self.filter.filter.clone(), + ) .await?; Ok(ReconciliationStep::ProcessDescendantBlocks(vec![block], 1)) } else { diff --git a/graph/src/data_source/mod.rs b/graph/src/data_source/mod.rs index 38a166710e0..e68b2b5c85d 100644 --- a/graph/src/data_source/mod.rs +++ b/graph/src/data_source/mod.rs @@ -258,7 +258,9 @@ impl DataSource { Ok(ds.match_and_decode(trigger)) } (Self::Onchain(_), TriggerData::Offchain(_)) - | (Self::Offchain(_), TriggerData::Onchain(_)) => Ok(None), + | (Self::Offchain(_), TriggerData::Onchain(_)) + | (Self::Onchain(_), TriggerData::Subgraph(_)) + | (Self::Offchain(_), TriggerData::Subgraph(_)) => Ok(None), (Self::Subgraph(_), _) => todo!(), // TODO(krishna) } } @@ -550,6 +552,7 @@ impl TriggerWithHandler { pub enum TriggerData { Onchain(C::TriggerData), Offchain(offchain::TriggerData), + Subgraph(subgraph::TriggerData), } impl TriggerData { @@ -557,6 +560,7 @@ impl TriggerData { match self { Self::Onchain(trigger) => trigger.error_context(), Self::Offchain(trigger) => format!("{:?}", trigger.source), + Self::Subgraph(trigger) => format!("{:?}", trigger.source), } } } diff --git a/graph/src/env/mod.rs b/graph/src/env/mod.rs index af53562528a..3b1ca5a2862 100644 --- a/graph/src/env/mod.rs +++ b/graph/src/env/mod.rs @@ -357,7 +357,7 @@ struct Inner { default = "false" )] allow_non_deterministic_fulltext_search: EnvVarBoolean, - #[envconfig(from = "GRAPH_MAX_SPEC_VERSION", default = "1.2.0")] + #[envconfig(from = "GRAPH_MAX_SPEC_VERSION", default = "1.3.0")] max_spec_version: Version, #[envconfig(from = "GRAPH_LOAD_WINDOW_SIZE", default = "300")] load_window_size_in_secs: u64, diff --git a/tests/src/fixture/ethereum.rs b/tests/src/fixture/ethereum.rs index b20672ce563..1004e4d8900 100644 --- a/tests/src/fixture/ethereum.rs +++ b/tests/src/fixture/ethereum.rs @@ -7,7 +7,7 @@ use super::{ NoopRuntimeAdapterBuilder, StaticBlockRefetcher, StaticStreamBuilder, Stores, TestChain, }; use graph::blockchain::client::ChainClient; -use graph::blockchain::{BlockPtr, TriggersAdapterSelector}; +use graph::blockchain::{BlockPtr, Trigger, TriggersAdapterSelector}; use graph::cheap_clone::CheapClone; use graph::prelude::ethabi::ethereum_types::H256; use graph::prelude::web3::types::{Address, Log, Transaction, H160}; @@ -81,7 +81,10 @@ pub fn genesis() -> BlockWithTriggers { number: Some(U64::from(ptr.number)), ..Default::default() })), - trigger_data: vec![EthereumTrigger::Block(ptr, EthereumBlockTriggerType::End)], + trigger_data: vec![Trigger::Chain(EthereumTrigger::Block( + ptr, + EthereumBlockTriggerType::End, + ))], } } @@ -128,7 +131,10 @@ pub fn empty_block(parent_ptr: BlockPtr, ptr: BlockPtr) -> BlockWithTriggers, payload: impl Into) { - block.trigger_data.push(EthereumTrigger::Block( - block.ptr(), - EthereumBlockTriggerType::End, - )) + block + .trigger_data + .push(Trigger::Chain(EthereumTrigger::Block( + block.ptr(), + EthereumBlockTriggerType::End, + ))) } From 19d22fdc214cdc43f67416f66be8f13969b64e0b Mon Sep 17 00:00:00 2001 From: incrypto32 Date: Tue, 16 Jul 2024 15:21:31 +0530 Subject: [PATCH 07/29] graph, chain: Build subgraph trigger filters in build_filter --- chain/ethereum/src/chain.rs | 2 +- core/src/subgraph/runner.rs | 16 +++++++++++++--- graph/src/blockchain/block_stream.rs | 11 ++++++++++- graph/src/blockchain/mod.rs | 15 ++++++--------- 4 files changed, 30 insertions(+), 14 deletions(-) diff --git a/chain/ethereum/src/chain.rs b/chain/ethereum/src/chain.rs index 46a9bc3184e..591bc9fdf6c 100644 --- a/chain/ethereum/src/chain.rs +++ b/chain/ethereum/src/chain.rs @@ -483,7 +483,7 @@ impl Blockchain for Chain { ) -> Result>, Error> { let current_ptr = store.block_ptr(); - if filter.subgraph_filter.is_some() { + if !filter.subgraph_filter.is_empty() { return self .block_stream_builder .build_subgraph_block_stream( diff --git a/core/src/subgraph/runner.rs b/core/src/subgraph/runner.rs index 1060fb2f93c..7c4e014b3e7 100644 --- a/core/src/subgraph/runner.rs +++ b/core/src/subgraph/runner.rs @@ -131,13 +131,23 @@ where None => true, }; + + let data_sources = self.ctx.static_data_sources(); + + let subgraph_filter = data_sources + .iter() + .filter_map(|ds| ds.as_subgraph()) + .map(|ds| (ds.source.address(), ds.source.start_block)) + .collect::>(); + + // if static_filters is not enabled we just stick to the filter based on all the data sources. if !static_filters { return TriggerFilterWrapper::new( C::TriggerFilter::from_data_sources( self.ctx.onchain_data_sources().filter(end_block_filter), ), - None, + subgraph_filter, ); } @@ -164,7 +174,7 @@ where filter.extend_with_template(templates.iter().filter_map(|ds| ds.as_onchain()).cloned()); - TriggerFilterWrapper::new(filter, None) + TriggerFilterWrapper::new(filter, subgraph_filter) } #[cfg(debug_assertions)] @@ -215,7 +225,7 @@ where let mut block_stream = new_block_stream( &self.inputs, - self.ctx.filter.cheap_clone().unwrap(), // Safe to unwrap as we just called `build_filter` in the previous line + self.ctx.filter.clone().unwrap(), // Safe to unwrap as we just called `build_filter` in the previous line &self.metrics.subgraph, ) .await? diff --git a/graph/src/blockchain/block_stream.rs b/graph/src/blockchain/block_stream.rs index 93693bb0384..8743fdd6c6b 100644 --- a/graph/src/blockchain/block_stream.rs +++ b/graph/src/blockchain/block_stream.rs @@ -308,7 +308,16 @@ impl TriggersAdapter for TriggersAdaterWrapper { to: BlockNumber, filter: &C::TriggerFilter, ) -> Result<(Vec>, BlockNumber), Error> { - self.adapter.scan_triggers(from, to, filter).await + // TODO(krishna): Do a proper implementation + self.adapter + .scan_triggers(from, to, filter) + .await + .map(|(mut blocks, next_block)| { + for _ in &mut blocks { + todo!() + } + (blocks, next_block) + }) } async fn triggers_in_block( diff --git a/graph/src/blockchain/mod.rs b/graph/src/blockchain/mod.rs index eccb0336b8e..746debb2f26 100644 --- a/graph/src/blockchain/mod.rs +++ b/graph/src/blockchain/mod.rs @@ -250,11 +250,14 @@ impl From for IngestorError { #[derive(Debug)] pub struct TriggerFilterWrapper { pub filter: Arc, - pub subgraph_filter: Option, + pub subgraph_filter: Vec<(DeploymentHash, BlockNumber)>, } impl TriggerFilterWrapper { - pub fn new(filter: C::TriggerFilter, subgraph_filter: Option) -> Self { + pub fn new( + filter: C::TriggerFilter, + subgraph_filter: Vec<(DeploymentHash, BlockNumber)>, + ) -> Self { Self { filter: Arc::new(filter), subgraph_filter, @@ -266,17 +269,11 @@ impl Clone for TriggerFilterWrapper { fn clone(&self) -> Self { Self { filter: self.filter.cheap_clone(), - subgraph_filter: self.subgraph_filter.cheap_clone(), + subgraph_filter: self.subgraph_filter.clone(), } } } -impl CheapClone for TriggerFilterWrapper { - fn cheap_clone(&self) -> Self { - self.clone() - } -} - pub trait TriggerFilter: Default + Clone + Send + Sync { fn from_data_sources<'a>( data_sources: impl Iterator + Clone, From af8cbfcc45460f7c9d864f579328a86e89153bfd Mon Sep 17 00:00:00 2001 From: incrypto32 Date: Wed, 17 Jul 2024 14:57:37 +0530 Subject: [PATCH 08/29] graph,core: Add subgraph_hosts to RuntimeHostBuilder --- core/src/subgraph/context/instance/hosts.rs | 2 +- core/src/subgraph/context/instance/mod.rs | 17 +++++++- core/src/subgraph/runner.rs | 6 +-- graph/src/blockchain/mod.rs | 2 +- graph/src/data_source/mod.rs | 10 +++-- runtime/wasm/src/module/mod.rs | 13 ++++++- .../subgraph-data-sources/src/mapping.ts | 17 ++------ .../subgraph-data-sources/subgraph.yaml | 2 +- tests/src/fixture/ethereum.rs | 18 ++++++++- tests/src/fixture/mod.rs | 39 ++++++++++++++----- tests/tests/runner_tests.rs | 34 ++++++++++++++-- 11 files changed, 120 insertions(+), 40 deletions(-) diff --git a/core/src/subgraph/context/instance/hosts.rs b/core/src/subgraph/context/instance/hosts.rs index 73701fadb29..f9a774246c8 100644 --- a/core/src/subgraph/context/instance/hosts.rs +++ b/core/src/subgraph/context/instance/hosts.rs @@ -57,7 +57,7 @@ impl> OnchainHosts { } pub fn push(&mut self, host: Arc) { - assert!(host.data_source().as_onchain().is_some()); + assert!(host.data_source().is_chain_based()); self.hosts.push(host.cheap_clone()); let idx = self.hosts.len() - 1; diff --git a/core/src/subgraph/context/instance/mod.rs b/core/src/subgraph/context/instance/mod.rs index fa723d3f18e..6436ea1a56e 100644 --- a/core/src/subgraph/context/instance/mod.rs +++ b/core/src/subgraph/context/instance/mod.rs @@ -29,6 +29,9 @@ pub(crate) struct SubgraphInstance> { /// will return the onchain hosts in the same order as they were inserted. onchain_hosts: OnchainHosts, + // TODO(krishna): Describe subgraph_hosts + subgraph_hosts: OnchainHosts, + offchain_hosts: OffchainHosts, /// Maps the hash of a module to a channel to the thread in which the module is instantiated. @@ -79,6 +82,7 @@ where network, static_data_sources: Arc::new(manifest.data_sources), onchain_hosts: OnchainHosts::new(), + subgraph_hosts: OnchainHosts::new(), offchain_hosts: OffchainHosts::new(), module_cache: HashMap::new(), templates, @@ -169,7 +173,14 @@ where Ok(Some(host)) } } - DataSource::Subgraph(_) => Ok(None), + DataSource::Subgraph(_) => { + if self.subgraph_hosts.contains(&host) { + Ok(None) + } else { + self.subgraph_hosts.push(host.cheap_clone()); + Ok(Some(host)) + } + } } } @@ -229,7 +240,9 @@ where TriggerData::Offchain(trigger) => self .offchain_hosts .matches_by_address(trigger.source.address().as_ref().map(|a| a.as_slice())), - TriggerData::Subgraph(_) => todo!(), // TODO(krishna) + TriggerData::Subgraph(trigger) => self + .subgraph_hosts + .matches_by_address(Some(trigger.source.to_bytes().as_slice())), } } diff --git a/core/src/subgraph/runner.rs b/core/src/subgraph/runner.rs index 7c4e014b3e7..a2d7eea4382 100644 --- a/core/src/subgraph/runner.rs +++ b/core/src/subgraph/runner.rs @@ -131,7 +131,6 @@ where None => true, }; - let data_sources = self.ctx.static_data_sources(); let subgraph_filter = data_sources @@ -140,7 +139,6 @@ where .map(|ds| (ds.source.address(), ds.source.start_block)) .collect::>(); - // if static_filters is not enabled we just stick to the filter based on all the data sources. if !static_filters { return TriggerFilterWrapper::new( @@ -341,7 +339,7 @@ where &block, triggers.into_iter().map(|t| match t { Trigger::Chain(t) => TriggerData::Onchain(t), - Trigger::Subgraph(_) => todo!(), //TODO(krishna), + Trigger::Subgraph(t) => TriggerData::Subgraph(t), }), hosts_filter, &self.metrics.subgraph, @@ -503,7 +501,7 @@ where &block, triggers.into_iter().map(|t| match t { Trigger::Chain(t) => TriggerData::Onchain(t), - Trigger::Subgraph(_) => todo!(), //TODO(krishna), + Trigger::Subgraph(_) => unreachable!(), // TODO(krishna): Re-evaulate this }), |_| Box::new(runtime_hosts.iter().map(Arc::as_ref)), &self.metrics.subgraph, diff --git a/graph/src/blockchain/mod.rs b/graph/src/blockchain/mod.rs index 746debb2f26..ebf958827e7 100644 --- a/graph/src/blockchain/mod.rs +++ b/graph/src/blockchain/mod.rs @@ -250,7 +250,7 @@ impl From for IngestorError { #[derive(Debug)] pub struct TriggerFilterWrapper { pub filter: Arc, - pub subgraph_filter: Vec<(DeploymentHash, BlockNumber)>, + pub subgraph_filter: Vec<(DeploymentHash, BlockNumber)>, // TODO(krishna): Make this a struct } impl TriggerFilterWrapper { diff --git a/graph/src/data_source/mod.rs b/graph/src/data_source/mod.rs index e68b2b5c85d..1d255b1563c 100644 --- a/graph/src/data_source/mod.rs +++ b/graph/src/data_source/mod.rs @@ -257,11 +257,15 @@ impl DataSource { (Self::Offchain(ds), TriggerData::Offchain(trigger)) => { Ok(ds.match_and_decode(trigger)) } + (Self::Subgraph(ds), TriggerData::Subgraph(trigger)) => { + Ok(ds.match_and_decode(block, trigger)) + } (Self::Onchain(_), TriggerData::Offchain(_)) | (Self::Offchain(_), TriggerData::Onchain(_)) | (Self::Onchain(_), TriggerData::Subgraph(_)) - | (Self::Offchain(_), TriggerData::Subgraph(_)) => Ok(None), - (Self::Subgraph(_), _) => todo!(), // TODO(krishna) + | (Self::Offchain(_), TriggerData::Subgraph(_)) + | (Self::Subgraph(_), TriggerData::Onchain(_)) + | (Self::Subgraph(_), TriggerData::Offchain(_)) => Ok(None), } } @@ -585,7 +589,7 @@ impl MappingTrigger { match self { Self::Onchain(trigger) => Some(trigger), Self::Offchain(_) => None, - Self::Subgraph(_) => todo!(), // TODO(krishna) + Self::Subgraph(_) => None, // TODO(krishna) } } } diff --git a/runtime/wasm/src/module/mod.rs b/runtime/wasm/src/module/mod.rs index ee19cd173aa..532f75d2660 100644 --- a/runtime/wasm/src/module/mod.rs +++ b/runtime/wasm/src/module/mod.rs @@ -4,6 +4,7 @@ use std::mem::MaybeUninit; use anyhow::anyhow; use anyhow::Error; use graph::blockchain::Blockchain; +use graph::data_source::subgraph; use graph::util::mem::init_slice; use semver::Version; use wasmtime::AsContext; @@ -69,6 +70,16 @@ impl ToAscPtr for offchain::TriggerData { } } +impl ToAscPtr for subgraph::TriggerData { + fn to_asc_ptr( + self, + heap: &mut H, + gas: &GasCounter, + ) -> Result, HostExportError> { + asc_new(heap, &self.entity.sorted_ref(), gas).map(|ptr| ptr.erase()) + } +} + impl ToAscPtr for MappingTrigger where C::MappingTrigger: ToAscPtr, @@ -81,7 +92,7 @@ where match self { MappingTrigger::Onchain(trigger) => trigger.to_asc_ptr(heap, gas), MappingTrigger::Offchain(trigger) => trigger.to_asc_ptr(heap, gas), - MappingTrigger::Subgraph(_) => todo!(), // TODO(krishna) + MappingTrigger::Subgraph(trigger) => trigger.to_asc_ptr(heap, gas), } } } diff --git a/tests/runner-tests/subgraph-data-sources/src/mapping.ts b/tests/runner-tests/subgraph-data-sources/src/mapping.ts index 3446d1f83c4..2e1a5382af3 100644 --- a/tests/runner-tests/subgraph-data-sources/src/mapping.ts +++ b/tests/runner-tests/subgraph-data-sources/src/mapping.ts @@ -1,15 +1,6 @@ -import { BigInt, dataSource, ethereum, log } from "@graphprotocol/graph-ts"; -import { Data } from "../generated/schema"; +import { Entity, log } from '@graphprotocol/graph-ts'; -export function handleBlock(block: ethereum.Block): void { - let foo = dataSource.context().getString("foo"); - let bar = dataSource.context().getI32("bar"); - let isTest = dataSource.context().getBoolean("isTest"); - if (block.number == BigInt.fromI32(0)) { - let data = new Data("0"); - data.foo = foo; - data.bar = bar; - data.isTest = isTest; - data.save(); - } +export function handleBlock(content: Entity): void { + let stringContent = content.getString('val'); + log.info('Content: {}', [stringContent]); } diff --git a/tests/runner-tests/subgraph-data-sources/subgraph.yaml b/tests/runner-tests/subgraph-data-sources/subgraph.yaml index b1a3fcbb486..050964f1c16 100644 --- a/tests/runner-tests/subgraph-data-sources/subgraph.yaml +++ b/tests/runner-tests/subgraph-data-sources/subgraph.yaml @@ -6,7 +6,7 @@ dataSources: name: Contract network: test source: - address: 'QmHash' + address: 'QmRFXhvyvbm4z5Lo7z2mN9Ckmo623uuB2jJYbRmAXgYKXJ' startBlock: 6082461 mapping: apiVersion: 0.0.7 diff --git a/tests/src/fixture/ethereum.rs b/tests/src/fixture/ethereum.rs index 1004e4d8900..5381a530148 100644 --- a/tests/src/fixture/ethereum.rs +++ b/tests/src/fixture/ethereum.rs @@ -9,9 +9,10 @@ use super::{ use graph::blockchain::client::ChainClient; use graph::blockchain::{BlockPtr, Trigger, TriggersAdapterSelector}; use graph::cheap_clone::CheapClone; +use graph::data_source::subgraph; use graph::prelude::ethabi::ethereum_types::H256; use graph::prelude::web3::types::{Address, Log, Transaction, H160}; -use graph::prelude::{ethabi, tiny_keccak, LightEthereumBlock, ENV_VARS}; +use graph::prelude::{ethabi, tiny_keccak, DeploymentHash, Entity, LightEthereumBlock, ENV_VARS}; use graph::{blockchain::block_stream::BlockWithTriggers, prelude::ethabi::ethereum_types::U64}; use graph_chain_ethereum::network::EthereumNetworkAdapters; use graph_chain_ethereum::trigger::LogRef; @@ -159,6 +160,21 @@ pub fn push_test_log(block: &mut BlockWithTriggers, payload: impl Into, + source: DeploymentHash, + entity: Entity, + entity_type: &str, +) { + block + .trigger_data + .push(Trigger::Subgraph(subgraph::TriggerData { + source, + entity: entity, + entity_type: entity_type.to_string(), + })); +} + pub fn push_test_command( block: &mut BlockWithTriggers, test_command: impl Into, diff --git a/tests/src/fixture/mod.rs b/tests/src/fixture/mod.rs index a6dbd650a3e..da37e8449f1 100644 --- a/tests/src/fixture/mod.rs +++ b/tests/src/fixture/mod.rs @@ -718,14 +718,25 @@ impl BlockStreamBuilder for MutexBlockStreamBuilder { async fn build_subgraph_block_stream( &self, - _chain: &C, - _deployment: DeploymentLocator, - _start_blocks: Vec, - _subgraph_current_block: Option, - _filter: Arc>, - _unified_api_version: graph::data::subgraph::UnifiedMappingApiVersion, + chain: &C, + deployment: DeploymentLocator, + start_blocks: Vec, + subgraph_current_block: Option, + filter: Arc>, + unified_api_version: graph::data::subgraph::UnifiedMappingApiVersion, ) -> anyhow::Result>> { - unimplemented!() + let builder = self.0.lock().unwrap().clone(); + + builder + .build_subgraph_block_stream( + chain, + deployment, + start_blocks, + subgraph_current_block, + filter, + unified_api_version, + ) + .await } async fn build_polling( @@ -772,11 +783,21 @@ where _chain: &C, _deployment: DeploymentLocator, _start_blocks: Vec, - _subgraph_current_block: Option, + subgraph_current_block: Option, _filter: Arc>, _unified_api_version: graph::data::subgraph::UnifiedMappingApiVersion, ) -> anyhow::Result>> { - unimplemented!() + let current_idx = subgraph_current_block.map(|current_block| { + self.chain + .iter() + .enumerate() + .find(|(_, b)| b.ptr() == current_block) + .unwrap() + .0 + }); + Ok(Box::new(StaticStream { + stream: Box::pin(stream_events(self.chain.clone(), current_idx)), + })) } async fn build_firehose( diff --git a/tests/tests/runner_tests.rs b/tests/tests/runner_tests.rs index 0ff9f40d537..eb409f6417c 100644 --- a/tests/tests/runner_tests.rs +++ b/tests/tests/runner_tests.rs @@ -18,11 +18,12 @@ use graph::object; use graph::prelude::ethabi::ethereum_types::H256; use graph::prelude::web3::types::Address; use graph::prelude::{ - hex, CheapClone, DeploymentHash, SubgraphAssignmentProvider, SubgraphName, SubgraphStore, + hex, CheapClone, DeploymentHash, SubgraphAssignmentProvider, SubgraphName, SubgraphStore, Value, }; +use graph::schema::InputSchema; use graph_tests::fixture::ethereum::{ chain, empty_block, generate_empty_blocks_for_range, genesis, push_test_command, push_test_log, - push_test_polling_trigger, + push_test_polling_trigger, push_test_subgraph_trigger, }; use graph_tests::fixture::substreams::chain as substreams_chain; @@ -1082,9 +1083,29 @@ async fn subgraph_data_sources() { let RunnerTestRecipe { stores, test_info } = RunnerTestRecipe::new("subgraph-data-sources", "subgraph-data-sources").await; + let schema = InputSchema::parse_latest( + "type User @entity { id: String!, val: String! }", + DeploymentHash::new("test").unwrap(), + ) + .unwrap(); + + let entity = schema + .make_entity(vec![ + ("id".into(), Value::String("id".to_owned())), + ("val".into(), Value::String("DATA".to_owned())), + ]) + .unwrap(); + let blocks = { let block_0 = genesis(); - let block_1 = empty_block(block_0.ptr(), test_ptr(1)); + let mut block_1 = empty_block(block_0.ptr(), test_ptr(1)); + push_test_subgraph_trigger( + &mut block_1, + DeploymentHash::new("QmRFXhvyvbm4z5Lo7z2mN9Ckmo623uuB2jJYbRmAXgYKXJ").unwrap(), + entity, + "User", + ); + let block_2 = empty_block(block_1.ptr(), test_ptr(2)); vec![block_0, block_1, block_2] }; @@ -1092,7 +1113,12 @@ async fn subgraph_data_sources() { let chain = chain(&test_info.test_name, blocks, &stores, None).await; let ctx = fixture::setup(&test_info, &stores, &chain, None, None).await; - ctx.start_and_sync_to(stop_block).await; + let _ = ctx + .runner(stop_block) + .await + .run_for_test(true) + .await + .unwrap(); } #[tokio::test] From 7691541be325daa7804d1750f2234935e64e14be Mon Sep 17 00:00:00 2001 From: incrypto32 Date: Thu, 18 Jul 2024 13:46:46 +0530 Subject: [PATCH 09/29] chain, core, graph: Add source_subgraph_stores to indexing inputs --- chain/arweave/src/chain.rs | 5 +- chain/cosmos/src/chain.rs | 5 +- chain/ethereum/src/chain.rs | 7 ++- chain/near/src/chain.rs | 6 +- chain/starknet/src/chain.rs | 7 ++- chain/substreams/src/block_stream.rs | 5 +- chain/substreams/src/chain.rs | 5 +- core/src/subgraph/inputs.rs | 5 +- core/src/subgraph/instance_manager.rs | 89 +++++++++++++++++++++++++++ core/src/subgraph/stream.rs | 1 + graph/src/blockchain/block_stream.rs | 3 +- graph/src/blockchain/mock.rs | 5 +- graph/src/blockchain/mod.rs | 3 +- tests/src/fixture/mod.rs | 11 +++- 14 files changed, 134 insertions(+), 23 deletions(-) diff --git a/chain/arweave/src/chain.rs b/chain/arweave/src/chain.rs index 9e6167b5678..4092eae3c5a 100644 --- a/chain/arweave/src/chain.rs +++ b/chain/arweave/src/chain.rs @@ -7,11 +7,11 @@ use graph::blockchain::{ }; use graph::cheap_clone::CheapClone; use graph::components::adapter::ChainId; -use graph::components::store::DeploymentCursorTracker; +use graph::components::store::{DeploymentCursorTracker, WritableStore}; use graph::data::subgraph::UnifiedMappingApiVersion; use graph::env::EnvVars; use graph::firehose::FirehoseEndpoint; -use graph::prelude::MetricsRegistry; +use graph::prelude::{DeploymentHash, MetricsRegistry}; use graph::substreams::Clock; use graph::{ blockchain::{ @@ -119,6 +119,7 @@ impl Blockchain for Chain { deployment: DeploymentLocator, store: impl DeploymentCursorTracker, start_blocks: Vec, + _source_subgraph_stores: Vec<(DeploymentHash, Arc)>, filter: Arc>, unified_api_version: UnifiedMappingApiVersion, ) -> Result>, Error> { diff --git a/chain/cosmos/src/chain.rs b/chain/cosmos/src/chain.rs index 868617968df..8eff2a9339c 100644 --- a/chain/cosmos/src/chain.rs +++ b/chain/cosmos/src/chain.rs @@ -2,7 +2,7 @@ use graph::blockchain::firehose_block_ingestor::FirehoseBlockIngestor; use graph::blockchain::{BlockIngestor, NoopDecoderHook, TriggerFilterWrapper}; use graph::components::adapter::ChainId; use graph::env::EnvVars; -use graph::prelude::MetricsRegistry; +use graph::prelude::{DeploymentHash, MetricsRegistry}; use graph::substreams::Clock; use std::convert::TryFrom; use std::sync::Arc; @@ -11,7 +11,7 @@ use graph::blockchain::block_stream::{BlockStreamError, BlockStreamMapper, Fireh use graph::blockchain::client::ChainClient; use graph::blockchain::{BasicBlockchainBuilder, BlockchainBuilder, NoopRuntimeAdapter}; use graph::cheap_clone::CheapClone; -use graph::components::store::DeploymentCursorTracker; +use graph::components::store::{DeploymentCursorTracker, WritableStore}; use graph::data::subgraph::UnifiedMappingApiVersion; use graph::{ blockchain::{ @@ -113,6 +113,7 @@ impl Blockchain for Chain { deployment: DeploymentLocator, store: impl DeploymentCursorTracker, start_blocks: Vec, + _source_subgraph_stores: Vec<(DeploymentHash, Arc)>, filter: Arc>, unified_api_version: UnifiedMappingApiVersion, ) -> Result>, Error> { diff --git a/chain/ethereum/src/chain.rs b/chain/ethereum/src/chain.rs index 591bc9fdf6c..f891acdacb6 100644 --- a/chain/ethereum/src/chain.rs +++ b/chain/ethereum/src/chain.rs @@ -7,12 +7,12 @@ use graph::blockchain::{ TriggersAdapterSelector, }; use graph::components::adapter::ChainId; -use graph::components::store::DeploymentCursorTracker; +use graph::components::store::{DeploymentCursorTracker, WritableStore}; use graph::data::subgraph::UnifiedMappingApiVersion; use graph::firehose::{FirehoseEndpoint, ForkStep}; use graph::futures03::compat::Future01CompatExt; use graph::prelude::{ - BlockHash, ComponentLoggerConfig, ElasticComponentLoggerConfig, EthereumBlock, + BlockHash, ComponentLoggerConfig, DeploymentHash, ElasticComponentLoggerConfig, EthereumBlock, EthereumCallCache, LightEthereumBlock, LightEthereumBlockExt, MetricsRegistry, }; use graph::schema::InputSchema; @@ -128,6 +128,7 @@ impl BlockStreamBuilder for EthereumStreamBuilder { chain: &Chain, deployment: DeploymentLocator, start_blocks: Vec, + _source_subgraph_stores: Vec<(DeploymentHash, Arc)>, subgraph_current_block: Option, filter: Arc>, unified_api_version: UnifiedMappingApiVersion, @@ -478,6 +479,7 @@ impl Blockchain for Chain { deployment: DeploymentLocator, store: impl DeploymentCursorTracker, start_blocks: Vec, + source_subgraph_stores: Vec<(DeploymentHash, Arc)>, filter: Arc>, unified_api_version: UnifiedMappingApiVersion, ) -> Result>, Error> { @@ -490,6 +492,7 @@ impl Blockchain for Chain { self, deployment, start_blocks, + source_subgraph_stores, current_ptr, filter, unified_api_version, diff --git a/chain/near/src/chain.rs b/chain/near/src/chain.rs index 3211870f069..b3203f40e33 100644 --- a/chain/near/src/chain.rs +++ b/chain/near/src/chain.rs @@ -8,12 +8,12 @@ use graph::blockchain::{ }; use graph::cheap_clone::CheapClone; use graph::components::adapter::ChainId; -use graph::components::store::DeploymentCursorTracker; +use graph::components::store::{DeploymentCursorTracker, WritableStore}; use graph::data::subgraph::UnifiedMappingApiVersion; use graph::env::EnvVars; use graph::firehose::FirehoseEndpoint; use graph::futures03::TryFutureExt; -use graph::prelude::MetricsRegistry; +use graph::prelude::{DeploymentHash, MetricsRegistry}; use graph::schema::InputSchema; use graph::substreams::{Clock, Package}; use graph::{ @@ -114,6 +114,7 @@ impl BlockStreamBuilder for NearStreamBuilder { _chain: &Chain, _deployment: DeploymentLocator, _start_blocks: Vec, + _source_subgraph_stores: Vec<(DeploymentHash, Arc)>, _subgraph_current_block: Option, _filter: Arc>, _unified_api_version: UnifiedMappingApiVersion, @@ -242,6 +243,7 @@ impl Blockchain for Chain { deployment: DeploymentLocator, store: impl DeploymentCursorTracker, start_blocks: Vec, + _source_subgraph_stores: Vec<(DeploymentHash, Arc)>, filter: Arc>, unified_api_version: UnifiedMappingApiVersion, ) -> Result>, Error> { diff --git a/chain/starknet/src/chain.rs b/chain/starknet/src/chain.rs index d926b31877b..5d7fdaf663d 100644 --- a/chain/starknet/src/chain.rs +++ b/chain/starknet/src/chain.rs @@ -16,15 +16,14 @@ use graph::{ cheap_clone::CheapClone, components::{ adapter::ChainId, - store::{DeploymentCursorTracker, DeploymentLocator}, + store::{DeploymentCursorTracker, DeploymentLocator, WritableStore}, }, data::subgraph::UnifiedMappingApiVersion, env::EnvVars, firehose::{self, FirehoseEndpoint, ForkStep}, futures03::future::TryFutureExt, prelude::{ - async_trait, BlockHash, BlockNumber, ChainStore, Error, Logger, LoggerFactory, - MetricsRegistry, + async_trait, BlockHash, BlockNumber, ChainStore, DeploymentHash, Error, Logger, LoggerFactory, MetricsRegistry }, schema::InputSchema, slog::o, @@ -115,6 +114,7 @@ impl Blockchain for Chain { deployment: DeploymentLocator, store: impl DeploymentCursorTracker, start_blocks: Vec, + _source_subgraph_stores: Vec<(DeploymentHash, Arc)>, filter: Arc>, unified_api_version: UnifiedMappingApiVersion, ) -> Result>, Error> { @@ -201,6 +201,7 @@ impl BlockStreamBuilder for StarknetStreamBuilder { _chain: &Chain, _deployment: DeploymentLocator, _start_blocks: Vec, + _source_subgraph_stores: Vec<(DeploymentHash, Arc)>, _subgraph_current_block: Option, _filter: Arc>, _unified_api_version: UnifiedMappingApiVersion, diff --git a/chain/substreams/src/block_stream.rs b/chain/substreams/src/block_stream.rs index 2d6eb902409..24cedade570 100644 --- a/chain/substreams/src/block_stream.rs +++ b/chain/substreams/src/block_stream.rs @@ -9,9 +9,9 @@ use graph::{ substreams_block_stream::SubstreamsBlockStream, Blockchain, TriggerFilterWrapper, }, - components::store::DeploymentLocator, + components::store::{DeploymentLocator, WritableStore}, data::subgraph::UnifiedMappingApiVersion, - prelude::{async_trait, BlockNumber, BlockPtr}, + prelude::{async_trait, BlockNumber, BlockPtr, DeploymentHash}, schema::InputSchema, slog::o, }; @@ -104,6 +104,7 @@ impl BlockStreamBuilderTrait for BlockStreamBuilder { _chain: &Chain, _deployment: DeploymentLocator, _start_blocks: Vec, + _source_subgraph_stores: Vec<(DeploymentHash, Arc)>, _subgraph_current_block: Option, _filter: Arc>, _unified_api_version: UnifiedMappingApiVersion, diff --git a/chain/substreams/src/chain.rs b/chain/substreams/src/chain.rs index 044617603d8..abb092c7499 100644 --- a/chain/substreams/src/chain.rs +++ b/chain/substreams/src/chain.rs @@ -7,9 +7,9 @@ use graph::blockchain::{ NoopRuntimeAdapter, TriggerFilterWrapper, }; use graph::components::adapter::ChainId; -use graph::components::store::DeploymentCursorTracker; +use graph::components::store::{DeploymentCursorTracker, WritableStore}; use graph::env::EnvVars; -use graph::prelude::{BlockHash, CheapClone, Entity, LoggerFactory, MetricsRegistry}; +use graph::prelude::{BlockHash, CheapClone, DeploymentHash, Entity, LoggerFactory, MetricsRegistry}; use graph::schema::EntityKey; use graph::{ blockchain::{ @@ -140,6 +140,7 @@ impl Blockchain for Chain { deployment: DeploymentLocator, store: impl DeploymentCursorTracker, _start_blocks: Vec, + _source_subgraph_stores: Vec<(DeploymentHash, Arc)>, filter: Arc>, _unified_api_version: UnifiedMappingApiVersion, ) -> Result>, Error> { diff --git a/core/src/subgraph/inputs.rs b/core/src/subgraph/inputs.rs index b2e95c753f5..3a12e33415f 100644 --- a/core/src/subgraph/inputs.rs +++ b/core/src/subgraph/inputs.rs @@ -6,7 +6,7 @@ use graph::{ }, data::subgraph::{SubgraphFeature, UnifiedMappingApiVersion}, data_source::DataSourceTemplate, - prelude::BlockNumber, + prelude::{BlockNumber, DeploymentHash}, }; use std::collections::BTreeSet; use std::sync::Arc; @@ -16,6 +16,7 @@ pub struct IndexingInputs { pub features: BTreeSet, pub start_blocks: Vec, pub end_blocks: BTreeSet, + pub source_subgraph_stores: Vec<(DeploymentHash, Arc)>, pub stop_block: Option, pub store: Arc, pub debug_fork: Option>, @@ -39,6 +40,7 @@ impl IndexingInputs { features, start_blocks, end_blocks, + source_subgraph_stores, stop_block, store: _, debug_fork, @@ -56,6 +58,7 @@ impl IndexingInputs { features: features.clone(), start_blocks: start_blocks.clone(), end_blocks: end_blocks.clone(), + source_subgraph_stores: source_subgraph_stores.clone(), stop_block: stop_block.clone(), store, debug_fork: debug_fork.clone(), diff --git a/core/src/subgraph/instance_manager.rs b/core/src/subgraph/instance_manager.rs index 75b0b86f81f..11ce5f065c7 100644 --- a/core/src/subgraph/instance_manager.rs +++ b/core/src/subgraph/instance_manager.rs @@ -9,12 +9,14 @@ use crate::subgraph::runner::SubgraphRunner; use graph::blockchain::block_stream::BlockStreamMetrics; use graph::blockchain::{Blockchain, BlockchainKind, DataSource, NodeCapabilities}; use graph::components::metrics::gas::GasMetrics; +use graph::components::store::WritableStore; use graph::components::subgraph::ProofOfIndexingVersion; use graph::data::subgraph::{UnresolvedSubgraphManifest, SPEC_VERSION_0_0_6}; use graph::data::value::Word; use graph::data_source::causality_region::CausalityRegionSeq; use graph::env::EnvVars; use graph::prelude::{SubgraphInstanceManager as SubgraphInstanceManagerTrait, *}; +use graph::semver::Version; use graph::{blockchain::BlockchainMap, components::store::DeploymentLocator}; use graph_runtime_wasm::module::ToAscPtr; use graph_runtime_wasm::RuntimeHostBuilder; @@ -202,6 +204,52 @@ impl SubgraphInstanceManager { } } + pub async fn hashes_to_writable_store( + &self, + logger: &Logger, + link_resolver: &Arc, + hashes: Vec, + max_spec_version: Version, + is_runner_test: bool, + ) -> anyhow::Result)>> { + let mut writable_stores = Vec::new(); + let subgraph_store = self.subgraph_store.clone(); + + if is_runner_test { + return Ok(writable_stores); + } + + for hash in hashes { + let file_bytes = link_resolver + .cat(logger, &hash.to_ipfs_link()) + .await + .map_err(SubgraphAssignmentProviderError::ResolveError)?; + let raw: serde_yaml::Mapping = serde_yaml::from_slice(&file_bytes) + .map_err(|e| SubgraphAssignmentProviderError::ResolveError(e.into()))?; + let manifest = UnresolvedSubgraphManifest::::parse(hash.cheap_clone(), raw)?; + let manifest = manifest + .resolve(&link_resolver, &logger, max_spec_version.clone()) + .await?; + + let loc = subgraph_store + .active_locator(&hash)? + .ok_or_else(|| anyhow!("no active deployment for hash {}", hash))?; + + let writable_store = subgraph_store + .clone() // Clone the Arc again for each iteration + .writable( + logger.clone(), + loc.id.clone(), + Arc::new(manifest.template_idx_and_name().collect()), + ) + .await?; + + writable_stores.push((loc.hash, writable_store)); + } + + Ok(writable_stores) + } + pub async fn build_subgraph_runner( &self, logger: Logger, @@ -211,6 +259,26 @@ impl SubgraphInstanceManager { stop_block: Option, tp: Box>>, ) -> anyhow::Result>> + where + C: Blockchain, + ::MappingTrigger: ToAscPtr, + { + self.build_subgraph_runner_inner( + logger, env_vars, deployment, manifest, stop_block, tp, false, + ) + .await + } + + pub async fn build_subgraph_runner_inner( + &self, + logger: Logger, + env_vars: Arc, + deployment: DeploymentLocator, + manifest: serde_yaml::Mapping, + stop_block: Option, + tp: Box>>, + is_runner_test: bool, + ) -> anyhow::Result>> where C: Blockchain, ::MappingTrigger: ToAscPtr, @@ -307,6 +375,16 @@ impl SubgraphInstanceManager { .filter_map(|d| d.as_onchain().cloned()) .collect::>(); + let subgraph_data_sources = data_sources + .iter() + .filter_map(|d| d.as_subgraph()) + .collect::>(); + + let subgraph_ds_source_deployments = subgraph_data_sources + .iter() + .map(|d| d.source.address()) + .collect::>(); + let required_capabilities = C::NodeCapabilities::from_data_sources(&onchain_data_sources); let network: Word = manifest.network_name().into(); @@ -413,11 +491,22 @@ impl SubgraphInstanceManager { let decoder = Box::new(Decoder::new(decoder_hook)); + let subgraph_data_source_writables = self + .hashes_to_writable_store::( + &logger, + &link_resolver, + subgraph_ds_source_deployments, + manifest.spec_version.clone(), + is_runner_test, + ) + .await?; + let inputs = IndexingInputs { deployment: deployment.clone(), features, start_blocks, end_blocks, + source_subgraph_stores: subgraph_data_source_writables, stop_block, store, debug_fork, diff --git a/core/src/subgraph/stream.rs b/core/src/subgraph/stream.rs index cfd41808e27..5547543f13d 100644 --- a/core/src/subgraph/stream.rs +++ b/core/src/subgraph/stream.rs @@ -18,6 +18,7 @@ pub async fn new_block_stream( inputs.deployment.clone(), inputs.store.cheap_clone(), inputs.start_blocks.clone(), + inputs.source_subgraph_stores.clone(), Arc::new(filter.clone()), inputs.unified_api_version.clone(), ) diff --git a/graph/src/blockchain/block_stream.rs b/graph/src/blockchain/block_stream.rs index 8743fdd6c6b..382378a9b24 100644 --- a/graph/src/blockchain/block_stream.rs +++ b/graph/src/blockchain/block_stream.rs @@ -14,7 +14,7 @@ use tokio::sync::mpsc::{self, Receiver, Sender}; use super::substreams_block_stream::SubstreamsLogData; use super::{Block, BlockPtr, BlockTime, Blockchain, Trigger, TriggerFilterWrapper}; use crate::anyhow::Result; -use crate::components::store::{BlockNumber, DeploymentLocator}; +use crate::components::store::{BlockNumber, DeploymentLocator, WritableStore}; use crate::data::subgraph::UnifiedMappingApiVersion; use crate::firehose::{self, FirehoseEndpoint}; use crate::futures03::stream::StreamExt as _; @@ -154,6 +154,7 @@ pub trait BlockStreamBuilder: Send + Sync { chain: &C, deployment: DeploymentLocator, start_blocks: Vec, + source_subgraph_stores: Vec<(DeploymentHash, Arc)>, subgraph_current_block: Option, filter: Arc>, unified_api_version: UnifiedMappingApiVersion, diff --git a/graph/src/blockchain/mock.rs b/graph/src/blockchain/mock.rs index e13a21878e4..4be185d1b39 100644 --- a/graph/src/blockchain/mock.rs +++ b/graph/src/blockchain/mock.rs @@ -2,11 +2,11 @@ use crate::{ bail, components::{ link_resolver::LinkResolver, - store::{BlockNumber, DeploymentCursorTracker, DeploymentLocator}, + store::{BlockNumber, DeploymentCursorTracker, DeploymentLocator, WritableStore}, subgraph::InstanceDSTemplateInfo, }, data::subgraph::UnifiedMappingApiVersion, - prelude::{BlockHash, DataSourceTemplateInfo}, + prelude::{BlockHash, DataSourceTemplateInfo, DeploymentHash}, }; use anyhow::Error; use async_trait::async_trait; @@ -373,6 +373,7 @@ impl Blockchain for MockBlockchain { _deployment: DeploymentLocator, _store: impl DeploymentCursorTracker, _start_blocks: Vec, + _source_subgraph_stores: Vec<(DeploymentHash, Arc)>, _filter: Arc>, _unified_api_version: UnifiedMappingApiVersion, ) -> Result>, Error> { diff --git a/graph/src/blockchain/mod.rs b/graph/src/blockchain/mod.rs index ebf958827e7..efb9d0659c4 100644 --- a/graph/src/blockchain/mod.rs +++ b/graph/src/blockchain/mod.rs @@ -20,7 +20,7 @@ use crate::{ components::{ adapter::ChainId, metrics::subgraph::SubgraphInstanceMetrics, - store::{DeploymentCursorTracker, DeploymentLocator, StoredDynamicDataSource}, + store::{DeploymentCursorTracker, DeploymentLocator, StoredDynamicDataSource, WritableStore}, subgraph::{HostMetrics, InstanceDSTemplateInfo, MappingError}, trigger_processor::RunnableTriggers, }, @@ -189,6 +189,7 @@ pub trait Blockchain: Debug + Sized + Send + Sync + Unpin + 'static { deployment: DeploymentLocator, store: impl DeploymentCursorTracker, start_blocks: Vec, + source_subgraph_stores: Vec<(DeploymentHash, Arc)>, filter: Arc>, unified_api_version: UnifiedMappingApiVersion, ) -> Result>, Error>; diff --git a/tests/src/fixture/mod.rs b/tests/src/fixture/mod.rs index da37e8449f1..095ca0446c7 100644 --- a/tests/src/fixture/mod.rs +++ b/tests/src/fixture/mod.rs @@ -20,7 +20,7 @@ use graph::cheap_clone::CheapClone; use graph::components::adapter::ChainId; use graph::components::link_resolver::{ArweaveClient, ArweaveResolver, FileSizeLimit}; use graph::components::metrics::MetricsRegistry; -use graph::components::store::{BlockStore, DeploymentLocator, EthereumCallCache}; +use graph::components::store::{BlockStore, DeploymentLocator, EthereumCallCache, WritableStore}; use graph::components::subgraph::Settings; use graph::data::graphql::load_manager::LoadManager; use graph::data::query::{Query, QueryTarget}; @@ -211,13 +211,14 @@ impl TestContext { let tp: Box> = Box::new(SubgraphTriggerProcessor {}); self.instance_manager - .build_subgraph_runner( + .build_subgraph_runner_inner( logger, self.env_vars.cheap_clone(), deployment, raw, Some(stop_block.block_number()), tp, + true, ) .await .unwrap() @@ -236,13 +237,14 @@ impl TestContext { ); self.instance_manager - .build_subgraph_runner( + .build_subgraph_runner_inner( logger, self.env_vars.cheap_clone(), deployment, raw, Some(stop_block.block_number()), tp, + true, ) .await .unwrap() @@ -721,6 +723,7 @@ impl BlockStreamBuilder for MutexBlockStreamBuilder { chain: &C, deployment: DeploymentLocator, start_blocks: Vec, + source_subgraph_stores: Vec<(DeploymentHash, Arc)>, subgraph_current_block: Option, filter: Arc>, unified_api_version: graph::data::subgraph::UnifiedMappingApiVersion, @@ -732,6 +735,7 @@ impl BlockStreamBuilder for MutexBlockStreamBuilder { chain, deployment, start_blocks, + source_subgraph_stores, subgraph_current_block, filter, unified_api_version, @@ -783,6 +787,7 @@ where _chain: &C, _deployment: DeploymentLocator, _start_blocks: Vec, + _source_subgraph_stores: Vec<(DeploymentHash, Arc)>, subgraph_current_block: Option, _filter: Arc>, _unified_api_version: graph::data::subgraph::UnifiedMappingApiVersion, From f61180101703f857829eb02e9db6cadfe85e49ff Mon Sep 17 00:00:00 2001 From: incrypto32 Date: Thu, 18 Jul 2024 15:28:41 +0530 Subject: [PATCH 10/29] graph, chain: minor refactoring and formatting --- chain/starknet/src/chain.rs | 3 ++- chain/substreams/src/chain.rs | 4 ++- graph/src/blockchain/block_stream.rs | 26 ++++++++++++++++++- graph/src/blockchain/mod.rs | 4 ++- .../subgraph-data-sources/subgraph.yaml | 2 +- 5 files changed, 34 insertions(+), 5 deletions(-) diff --git a/chain/starknet/src/chain.rs b/chain/starknet/src/chain.rs index 5d7fdaf663d..865ad90af10 100644 --- a/chain/starknet/src/chain.rs +++ b/chain/starknet/src/chain.rs @@ -23,7 +23,8 @@ use graph::{ firehose::{self, FirehoseEndpoint, ForkStep}, futures03::future::TryFutureExt, prelude::{ - async_trait, BlockHash, BlockNumber, ChainStore, DeploymentHash, Error, Logger, LoggerFactory, MetricsRegistry + async_trait, BlockHash, BlockNumber, ChainStore, DeploymentHash, Error, Logger, + LoggerFactory, MetricsRegistry, }, schema::InputSchema, slog::o, diff --git a/chain/substreams/src/chain.rs b/chain/substreams/src/chain.rs index abb092c7499..b3cf8cca8a6 100644 --- a/chain/substreams/src/chain.rs +++ b/chain/substreams/src/chain.rs @@ -9,7 +9,9 @@ use graph::blockchain::{ use graph::components::adapter::ChainId; use graph::components::store::{DeploymentCursorTracker, WritableStore}; use graph::env::EnvVars; -use graph::prelude::{BlockHash, CheapClone, DeploymentHash, Entity, LoggerFactory, MetricsRegistry}; +use graph::prelude::{ + BlockHash, CheapClone, DeploymentHash, Entity, LoggerFactory, MetricsRegistry, +}; use graph::schema::EntityKey; use graph::{ blockchain::{ diff --git a/graph/src/blockchain/block_stream.rs b/graph/src/blockchain/block_stream.rs index 382378a9b24..5784307a0b1 100644 --- a/graph/src/blockchain/block_stream.rs +++ b/graph/src/blockchain/block_stream.rs @@ -303,13 +303,13 @@ impl TriggersAdapter for TriggersAdaterWrapper { self.adapter.ancestor_block(ptr, offset, root).await } + // TODO: Do a proper implementation, this is a complete mock implementation async fn scan_triggers( &self, from: BlockNumber, to: BlockNumber, filter: &C::TriggerFilter, ) -> Result<(Vec>, BlockNumber), Error> { - // TODO(krishna): Do a proper implementation self.adapter .scan_triggers(from, to, filter) .await @@ -339,6 +339,30 @@ impl TriggersAdapter for TriggersAdaterWrapper { } } +// fn create_mock_trigger() -> Trigger { +// let entity = create_mock_entity(); +// Trigger::Subgraph(subgraph::TriggerData { +// source: DeploymentHash::new("test").unwrap(), +// entity, +// entity_type: "User".to_string(), +// }) +// } + +// fn create_mock_entity() -> Entity { +// let schema = InputSchema::parse_latest( +// "type User @entity { id: String!, val: String! }", +// DeploymentHash::new("test").unwrap(), +// ) +// .unwrap(); + +// schema +// .make_entity(vec![ +// ("id".into(), Value::String("id".to_owned())), +// ("val".into(), Value::String("content".to_owned())), +// ]) +// .unwrap() +// } + #[async_trait] pub trait TriggersAdapter: Send + Sync { // Return the block that is `offset` blocks before the block pointed to by `ptr` from the local diff --git a/graph/src/blockchain/mod.rs b/graph/src/blockchain/mod.rs index efb9d0659c4..14dceaf45e1 100644 --- a/graph/src/blockchain/mod.rs +++ b/graph/src/blockchain/mod.rs @@ -20,7 +20,9 @@ use crate::{ components::{ adapter::ChainId, metrics::subgraph::SubgraphInstanceMetrics, - store::{DeploymentCursorTracker, DeploymentLocator, StoredDynamicDataSource, WritableStore}, + store::{ + DeploymentCursorTracker, DeploymentLocator, StoredDynamicDataSource, WritableStore, + }, subgraph::{HostMetrics, InstanceDSTemplateInfo, MappingError}, trigger_processor::RunnableTriggers, }, diff --git a/tests/runner-tests/subgraph-data-sources/subgraph.yaml b/tests/runner-tests/subgraph-data-sources/subgraph.yaml index 050964f1c16..1c666e3417e 100644 --- a/tests/runner-tests/subgraph-data-sources/subgraph.yaml +++ b/tests/runner-tests/subgraph-data-sources/subgraph.yaml @@ -15,5 +15,5 @@ dataSources: - Gravatar handlers: - handler: handleBlock - entity: Gravatar + entity: User file: ./src/mapping.ts From 1afe3fc035661b6c6e08e43345581d8513b351cd Mon Sep 17 00:00:00 2001 From: incrypto32 Date: Thu, 18 Jul 2024 16:37:59 +0530 Subject: [PATCH 11/29] graph, chain: fix typo in TriggersAdapterWrapper --- chain/ethereum/src/chain.rs | 4 ++-- graph/src/blockchain/block_stream.rs | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/chain/ethereum/src/chain.rs b/chain/ethereum/src/chain.rs index f891acdacb6..18ea14b82b7 100644 --- a/chain/ethereum/src/chain.rs +++ b/chain/ethereum/src/chain.rs @@ -62,7 +62,7 @@ use crate::{BufferedCallCache, NodeCapabilities}; use crate::{EthereumAdapter, RuntimeAdapter}; use graph::blockchain::block_stream::{ BlockStream, BlockStreamBuilder, BlockStreamError, BlockStreamMapper, FirehoseCursor, - TriggersAdaterWrapper, + TriggersAdapterWrapper, }; /// Celo Mainnet: 42220, Testnet Alfajores: 44787, Testnet Baklava: 62320 @@ -143,7 +143,7 @@ impl BlockStreamBuilder for EthereumStreamBuilder { ) }); - let adapter = Arc::new(TriggersAdaterWrapper::new(adapter)); + let adapter = Arc::new(TriggersAdapterWrapper::new(adapter)); let logger = chain .logger_factory diff --git a/graph/src/blockchain/block_stream.rs b/graph/src/blockchain/block_stream.rs index 5784307a0b1..190445e85bd 100644 --- a/graph/src/blockchain/block_stream.rs +++ b/graph/src/blockchain/block_stream.rs @@ -282,18 +282,18 @@ impl BlockWithTriggers { } } -pub struct TriggersAdaterWrapper { +pub struct TriggersAdapterWrapper { pub adapter: Arc>, } -impl TriggersAdaterWrapper { +impl TriggersAdapterWrapper { pub fn new(adapter: Arc>) -> Self { Self { adapter } } } #[async_trait] -impl TriggersAdapter for TriggersAdaterWrapper { +impl TriggersAdapter for TriggersAdapterWrapper { async fn ancestor_block( &self, ptr: BlockPtr, From abd65fc6a63811f4259554d7fa91c20528d1b3a5 Mon Sep 17 00:00:00 2001 From: incrypto32 Date: Thu, 18 Jul 2024 16:55:03 +0530 Subject: [PATCH 12/29] chain, graph: use TriggerFilterWrapper in scan_triggers --- chain/arweave/src/chain.rs | 2 +- chain/cosmos/src/chain.rs | 2 +- chain/ethereum/src/chain.rs | 8 ++++---- chain/near/src/chain.rs | 2 +- chain/starknet/src/chain.rs | 2 +- chain/substreams/src/trigger.rs | 5 +++-- graph/src/blockchain/block_stream.rs | 15 +++++++++++---- graph/src/blockchain/mock.rs | 4 ++-- graph/src/blockchain/polling_block_stream.rs | 5 +---- tests/src/fixture/mod.rs | 2 +- 10 files changed, 26 insertions(+), 21 deletions(-) diff --git a/chain/arweave/src/chain.rs b/chain/arweave/src/chain.rs index 4092eae3c5a..163aed06204 100644 --- a/chain/arweave/src/chain.rs +++ b/chain/arweave/src/chain.rs @@ -198,7 +198,7 @@ impl TriggersAdapterTrait for TriggersAdapter { &self, _from: BlockNumber, _to: BlockNumber, - _filter: &TriggerFilter, + _filter: &Arc>, ) -> Result<(Vec>, BlockNumber), Error> { panic!("Should never be called since not used by FirehoseBlockStream") } diff --git a/chain/cosmos/src/chain.rs b/chain/cosmos/src/chain.rs index 8eff2a9339c..7a20adc127f 100644 --- a/chain/cosmos/src/chain.rs +++ b/chain/cosmos/src/chain.rs @@ -201,7 +201,7 @@ impl TriggersAdapterTrait for TriggersAdapter { &self, _from: BlockNumber, _to: BlockNumber, - _filter: &TriggerFilter, + _filter: &Arc>, ) -> Result<(Vec>, BlockNumber), Error> { panic!("Should never be called since not used by FirehoseBlockStream") } diff --git a/chain/ethereum/src/chain.rs b/chain/ethereum/src/chain.rs index 18ea14b82b7..9dffc477f5c 100644 --- a/chain/ethereum/src/chain.rs +++ b/chain/ethereum/src/chain.rs @@ -128,7 +128,7 @@ impl BlockStreamBuilder for EthereumStreamBuilder { chain: &Chain, deployment: DeploymentLocator, start_blocks: Vec, - _source_subgraph_stores: Vec<(DeploymentHash, Arc)>, + source_subgraph_stores: Vec<(DeploymentHash, Arc)>, subgraph_current_block: Option, filter: Arc>, unified_api_version: UnifiedMappingApiVersion, @@ -143,7 +143,7 @@ impl BlockStreamBuilder for EthereumStreamBuilder { ) }); - let adapter = Arc::new(TriggersAdapterWrapper::new(adapter)); + let adapter = Arc::new(TriggersAdapterWrapper::new(adapter, source_subgraph_stores)); let logger = chain .logger_factory @@ -758,7 +758,7 @@ impl TriggersAdapterTrait for TriggersAdapter { &self, from: BlockNumber, to: BlockNumber, - filter: &TriggerFilter, + filter: &Arc>, ) -> Result<(Vec>, BlockNumber), Error> { blocks_with_triggers( self.chain_client @@ -770,7 +770,7 @@ impl TriggersAdapterTrait for TriggersAdapter { self.ethrpc_metrics.clone(), from, to, - filter, + &filter.filter.clone(), self.unified_api_version.clone(), ) .await diff --git a/chain/near/src/chain.rs b/chain/near/src/chain.rs index b3203f40e33..73a17a6640a 100644 --- a/chain/near/src/chain.rs +++ b/chain/near/src/chain.rs @@ -331,7 +331,7 @@ impl TriggersAdapterTrait for TriggersAdapter { &self, _from: BlockNumber, _to: BlockNumber, - _filter: &TriggerFilter, + _filter: &Arc>, ) -> Result<(Vec>, BlockNumber), Error> { panic!("Should never be called since not used by FirehoseBlockStream") } diff --git a/chain/starknet/src/chain.rs b/chain/starknet/src/chain.rs index 865ad90af10..620c8412760 100644 --- a/chain/starknet/src/chain.rs +++ b/chain/starknet/src/chain.rs @@ -393,7 +393,7 @@ impl TriggersAdapterTrait for TriggersAdapter { &self, _from: BlockNumber, _to: BlockNumber, - _filter: &crate::adapter::TriggerFilter, + _filter: &Arc>, ) -> Result<(Vec>, BlockNumber), Error> { panic!("Should never be called since not used by FirehoseBlockStream") } diff --git a/chain/substreams/src/trigger.rs b/chain/substreams/src/trigger.rs index 2b47e4e57b8..5f919a12a90 100644 --- a/chain/substreams/src/trigger.rs +++ b/chain/substreams/src/trigger.rs @@ -1,7 +1,8 @@ use anyhow::Error; use graph::{ blockchain::{ - self, block_stream::BlockWithTriggers, BlockPtr, EmptyNodeCapabilities, MappingTriggerTrait, + self, block_stream::BlockWithTriggers, BlockPtr, EmptyNodeCapabilities, + MappingTriggerTrait, TriggerFilterWrapper, }, components::{ store::{DeploymentLocator, SubgraphFork}, @@ -140,7 +141,7 @@ impl blockchain::TriggersAdapter for TriggersAdapter { &self, _from: BlockNumber, _to: BlockNumber, - _filter: &TriggerFilter, + _filter: &Arc>, ) -> Result<(Vec>, BlockNumber), Error> { unimplemented!() } diff --git a/graph/src/blockchain/block_stream.rs b/graph/src/blockchain/block_stream.rs index 190445e85bd..3979c28ce92 100644 --- a/graph/src/blockchain/block_stream.rs +++ b/graph/src/blockchain/block_stream.rs @@ -284,11 +284,18 @@ impl BlockWithTriggers { pub struct TriggersAdapterWrapper { pub adapter: Arc>, + pub source_subgraph_stores: Vec<(DeploymentHash, Arc)>, } impl TriggersAdapterWrapper { - pub fn new(adapter: Arc>) -> Self { - Self { adapter } + pub fn new( + adapter: Arc>, + source_subgraph_stores: Vec<(DeploymentHash, Arc)>, + ) -> Self { + Self { + adapter, + source_subgraph_stores, + } } } @@ -308,7 +315,7 @@ impl TriggersAdapter for TriggersAdapterWrapper { &self, from: BlockNumber, to: BlockNumber, - filter: &C::TriggerFilter, + filter: &Arc>, ) -> Result<(Vec>, BlockNumber), Error> { self.adapter .scan_triggers(from, to, filter) @@ -386,7 +393,7 @@ pub trait TriggersAdapter: Send + Sync { &self, from: BlockNumber, to: BlockNumber, - filter: &C::TriggerFilter, + filter: &Arc>, ) -> Result<(Vec>, BlockNumber), Error>; // Used for reprocessing blocks when creating a data source. diff --git a/graph/src/blockchain/mock.rs b/graph/src/blockchain/mock.rs index 4be185d1b39..c08f688a4b4 100644 --- a/graph/src/blockchain/mock.rs +++ b/graph/src/blockchain/mock.rs @@ -232,7 +232,7 @@ impl TriggersAdapter for MockTriggersAdapter { &self, from: crate::components::store::BlockNumber, to: crate::components::store::BlockNumber, - filter: &MockTriggerFilter, + filter: &Arc>, ) -> Result< ( Vec>, @@ -264,7 +264,7 @@ impl TriggersAdapter for MockTriggersAdapter { async fn blocks_with_triggers( _from: crate::components::store::BlockNumber, to: crate::components::store::BlockNumber, - _filter: &MockTriggerFilter, + _filter: &Arc>, ) -> Result< ( Vec>, diff --git a/graph/src/blockchain/polling_block_stream.rs b/graph/src/blockchain/polling_block_stream.rs index 64a84ff3b94..8cf466a6486 100644 --- a/graph/src/blockchain/polling_block_stream.rs +++ b/graph/src/blockchain/polling_block_stream.rs @@ -379,10 +379,7 @@ where ); // Update with actually scanned range, to account for any skipped null blocks. - let (blocks, to) = self - .adapter - .scan_triggers(from, to, &self.filter.filter.clone()) - .await?; + let (blocks, to) = self.adapter.scan_triggers(from, to, &self.filter).await?; let range_size = to - from + 1; // If the target block (`to`) is within the reorg threshold, indicating no non-null finalized blocks are diff --git a/tests/src/fixture/mod.rs b/tests/src/fixture/mod.rs index 095ca0446c7..358c5a8e425 100644 --- a/tests/src/fixture/mod.rs +++ b/tests/src/fixture/mod.rs @@ -1006,7 +1006,7 @@ impl TriggersAdapter for MockTriggersAdapter { &self, _from: BlockNumber, _to: BlockNumber, - _filter: &::TriggerFilter, + _filter: &Arc>, ) -> Result<(Vec>, BlockNumber), Error> { todo!() } From b9664aee1dcdd664207af2c5a5145835d0ad59b2 Mon Sep 17 00:00:00 2001 From: incrypto32 Date: Wed, 24 Jul 2024 09:56:51 +0530 Subject: [PATCH 13/29] chain/ethereum: implement load_blocks_by_numbers for EthereumAdapter --- chain/ethereum/src/adapter.rs | 7 +++++++ chain/ethereum/src/ethereum_adapter.rs | 22 ++++++++++++++++++++++ graph/src/blockchain/types.rs | 4 ++++ 3 files changed, 33 insertions(+) diff --git a/chain/ethereum/src/adapter.rs b/chain/ethereum/src/adapter.rs index f78ff1b0bec..31083724809 100644 --- a/chain/ethereum/src/adapter.rs +++ b/chain/ethereum/src/adapter.rs @@ -1109,6 +1109,13 @@ pub trait EthereumAdapter: Send + Sync + 'static { block_hash: H256, ) -> Box + Send>; + async fn load_blocks_by_numbers( + &self, + logger: Logger, + chain_store: Arc, + block_numbers: HashSet, + ) -> Box, Error = Error> + Send>; + /// Load Ethereum blocks in bulk, returning results as they come back as a Stream. /// May use the `chain_store` as a cache. async fn load_blocks( diff --git a/chain/ethereum/src/ethereum_adapter.rs b/chain/ethereum/src/ethereum_adapter.rs index 123f79bb4a8..7c08c61950e 100644 --- a/chain/ethereum/src/ethereum_adapter.rs +++ b/chain/ethereum/src/ethereum_adapter.rs @@ -1648,6 +1648,28 @@ impl EthereumAdapterTrait for EthereumAdapter { Ok(decoded) } + // This is a ugly temporary implementation to get the block ptrs for a range of blocks + async fn load_blocks_by_numbers( + &self, + logger: Logger, + chain_store: Arc, + block_numbers: HashSet, + ) -> Box, Error = Error> + Send> { + let block_hashes = block_numbers + .into_iter() + .map(|number| { + chain_store + .block_hashes_by_block_number(number) + .unwrap() + .first() + .unwrap() + .as_h256() + }) + .collect::>(); + + self.load_blocks(logger, chain_store, block_hashes).await + } + /// Load Ethereum blocks in bulk, returning results as they come back as a Stream. async fn load_blocks( &self, diff --git a/graph/src/blockchain/types.rs b/graph/src/blockchain/types.rs index 931e52e2dd5..7c670d4cdd6 100644 --- a/graph/src/blockchain/types.rs +++ b/graph/src/blockchain/types.rs @@ -31,6 +31,10 @@ impl BlockHash { &self.0 } + pub fn as_h256(&self) -> H256 { + H256::from_slice(self.as_slice()) + } + /// Encodes the block hash into a hexadecimal string **without** a "0x" /// prefix. Hashes are stored in the database in this format when the /// schema uses `text` columns, which is a legacy and such columns From f7fe8049506b9b06d65ab9c7692d6f67c4f6dbcb Mon Sep 17 00:00:00 2001 From: incrypto32 Date: Wed, 24 Jul 2024 12:53:31 +0530 Subject: [PATCH 14/29] graph: refactor BlockWithTriggers impl --- graph/src/blockchain/block_stream.rs | 41 +++++++++++++++++----------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/graph/src/blockchain/block_stream.rs b/graph/src/blockchain/block_stream.rs index 3979c28ce92..a528bc97c15 100644 --- a/graph/src/blockchain/block_stream.rs +++ b/graph/src/blockchain/block_stream.rs @@ -1,3 +1,4 @@ +use crate::data_source::subgraph; use crate::substreams::Clock; use crate::substreams_rpc::response::Message as SubstreamsMessage; use crate::substreams_rpc::BlockScopedData; @@ -228,14 +229,30 @@ impl BlockWithTriggers { /// Creates a BlockWithTriggers structure, which holds /// the trigger data ordered and without any duplicates. pub fn new(block: C::Block, trigger_data: Vec, logger: &Logger) -> Self { - let mut trigger_data = trigger_data - .into_iter() - .map(|trigger_data| { - let trigger = Trigger::Chain(trigger_data); - trigger - }) - .collect::>(); + Self::new_with_triggers( + block, + trigger_data.into_iter().map(Trigger::Chain).collect(), + logger, + ) + } + + pub fn new_with_subgraph_triggers( + block: C::Block, + trigger_data: Vec, + logger: &Logger, + ) -> Self { + Self::new_with_triggers( + block, + trigger_data.into_iter().map(Trigger::Subgraph).collect(), + logger, + ) + } + fn new_with_triggers( + block: C::Block, + mut trigger_data: Vec>, + logger: &Logger, + ) -> Self { // This is where triggers get sorted. trigger_data.sort(); @@ -317,15 +334,7 @@ impl TriggersAdapter for TriggersAdapterWrapper { to: BlockNumber, filter: &Arc>, ) -> Result<(Vec>, BlockNumber), Error> { - self.adapter - .scan_triggers(from, to, filter) - .await - .map(|(mut blocks, next_block)| { - for _ in &mut blocks { - todo!() - } - (blocks, next_block) - }) + self.adapter.scan_triggers(from, to, filter).await } async fn triggers_in_block( From 3d39bde4fd7bf2f137e827541770b8d9e298092b Mon Sep 17 00:00:00 2001 From: incrypto32 Date: Wed, 24 Jul 2024 12:55:05 +0530 Subject: [PATCH 15/29] graph, core: Add a new SubgraphFilter struct --- core/src/subgraph/runner.rs | 13 +++++++++++-- graph/src/blockchain/mod.rs | 12 ++++++++++-- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/core/src/subgraph/runner.rs b/core/src/subgraph/runner.rs index a2d7eea4382..9712a9095fe 100644 --- a/core/src/subgraph/runner.rs +++ b/core/src/subgraph/runner.rs @@ -8,7 +8,7 @@ use graph::blockchain::block_stream::{ BlockStreamError, BlockStreamEvent, BlockWithTriggers, FirehoseCursor, }; use graph::blockchain::{ - Block, BlockTime, Blockchain, DataSource as _, Trigger, TriggerFilter as _, + Block, BlockTime, Blockchain, DataSource as _, SubgraphFilter, Trigger, TriggerFilter as _, TriggerFilterWrapper, }; use graph::components::store::{EmptyStore, GetScope, ReadStore, StoredDynamicDataSource}; @@ -136,7 +136,16 @@ where let subgraph_filter = data_sources .iter() .filter_map(|ds| ds.as_subgraph()) - .map(|ds| (ds.source.address(), ds.source.start_block)) + .map(|ds| SubgraphFilter { + subgraph: ds.source.address(), + start_block: ds.source.start_block, + entities: ds + .mapping + .handlers + .iter() + .map(|handler| handler.entity.clone()) + .collect(), + }) .collect::>(); // if static_filters is not enabled we just stick to the filter based on all the data sources. diff --git a/graph/src/blockchain/mod.rs b/graph/src/blockchain/mod.rs index 14dceaf45e1..e6c410acc3a 100644 --- a/graph/src/blockchain/mod.rs +++ b/graph/src/blockchain/mod.rs @@ -253,13 +253,21 @@ impl From for IngestorError { #[derive(Debug)] pub struct TriggerFilterWrapper { pub filter: Arc, - pub subgraph_filter: Vec<(DeploymentHash, BlockNumber)>, // TODO(krishna): Make this a struct + pub subgraph_filter: Vec, +} + + +#[derive(Clone, Debug)] +pub struct SubgraphFilter { + pub subgraph: DeploymentHash, + pub start_block: BlockNumber, + pub entities: Vec, } impl TriggerFilterWrapper { pub fn new( filter: C::TriggerFilter, - subgraph_filter: Vec<(DeploymentHash, BlockNumber)>, + subgraph_filter: Vec, ) -> Self { Self { filter: Arc::new(filter), From c1eb1d13e34ca7369824781df04faa1fa9baaf38 Mon Sep 17 00:00:00 2001 From: incrypto32 Date: Wed, 24 Jul 2024 12:57:09 +0530 Subject: [PATCH 16/29] chain/ethereum: Mock implementation of subgraph_triggers for ethereum --- chain/ethereum/src/chain.rs | 4 +- chain/ethereum/src/ethereum_adapter.rs | 106 ++++++++++++++++++++++++- graph/src/blockchain/mod.rs | 6 +- 3 files changed, 107 insertions(+), 9 deletions(-) diff --git a/chain/ethereum/src/chain.rs b/chain/ethereum/src/chain.rs index 9dffc477f5c..b5a507565a7 100644 --- a/chain/ethereum/src/chain.rs +++ b/chain/ethereum/src/chain.rs @@ -770,7 +770,7 @@ impl TriggersAdapterTrait for TriggersAdapter { self.ethrpc_metrics.clone(), from, to, - &filter.filter.clone(), + filter, self.unified_api_version.clone(), ) .await @@ -807,7 +807,7 @@ impl TriggersAdapterTrait for TriggersAdapter { self.ethrpc_metrics.clone(), block_number, block_number, - filter, + &Arc::new(TriggerFilterWrapper::::new(filter.clone(), vec![])), // TODO(krishna): This is temporary until we take TriggerFilterWrapper as param in triggers_in_block self.unified_api_version.clone(), ) .await?; diff --git a/chain/ethereum/src/ethereum_adapter.rs b/chain/ethereum/src/ethereum_adapter.rs index 7c08c61950e..7e9786a6883 100644 --- a/chain/ethereum/src/ethereum_adapter.rs +++ b/chain/ethereum/src/ethereum_adapter.rs @@ -2,11 +2,14 @@ use futures03::{future::BoxFuture, stream::FuturesUnordered}; use graph::blockchain::client::ChainClient; use graph::blockchain::BlockHash; use graph::blockchain::ChainIdentifier; +use graph::blockchain::SubgraphFilter; +use graph::blockchain::TriggerFilterWrapper; use graph::components::transaction_receipt::LightTransactionReceipt; use graph::data::store::ethereum::call; use graph::data::store::scalar; use graph::data::subgraph::UnifiedMappingApiVersion; use graph::data::subgraph::API_VERSION_0_0_7; +use graph::data_source::subgraph; use graph::futures01::stream; use graph::futures01::Future; use graph::futures01::Stream; @@ -18,6 +21,10 @@ use graph::prelude::ethabi::ParamType; use graph::prelude::ethabi::Token; use graph::prelude::tokio::try_join; use graph::prelude::web3::types::U256; +use graph::prelude::DeploymentHash; +use graph::prelude::Entity; +use graph::prelude::Value; +use graph::schema::InputSchema; use graph::slog::o; use graph::tokio::sync::RwLock; use graph::tokio::time::timeout; @@ -66,7 +73,7 @@ use crate::{ }, transport::Transport, trigger::{EthereumBlockTriggerType, EthereumTrigger}, - TriggerFilter, ENV_VARS, + ENV_VARS, }; #[derive(Debug, Clone)] @@ -1722,6 +1729,81 @@ impl EthereumAdapterTrait for EthereumAdapter { } } +// TODO(krishna): Currently this is a mock implementation of subgraph triggers. +// This will be replaced with the actual implementation which will use the filters to +// query the database of the source subgraph and return the entity triggers. +async fn subgraph_triggers( + adapter: Arc, + logger: Logger, + chain_store: Arc, + _subgraph_metrics: Arc, + from: BlockNumber, + to: BlockNumber, + filter: &Arc>, + _unified_api_version: UnifiedMappingApiVersion, +) -> Result<(Vec>, BlockNumber), Error> { + let logger2 = logger.cheap_clone(); + let eth = adapter.clone(); + let to_ptr = eth.next_existing_ptr_to_number(&logger, to).await?; + let to = to_ptr.block_number(); + + let first_filter = filter.subgraph_filter.first().unwrap(); + + let blocks = adapter + .load_blocks_by_numbers( + logger.cheap_clone(), + chain_store.clone(), + HashSet::from_iter(from..=to), + ) + .await + .and_then(move |block| { + Ok(BlockWithTriggers::::new_with_subgraph_triggers( + BlockFinality::Final(block.clone()), + vec![create_mock_subgraph_trigger(first_filter, &block)], + &logger2, + )) + }) + .collect() + .compat() + .await?; + + Ok((blocks, to)) +} + +fn create_mock_subgraph_trigger( + filter: &SubgraphFilter, + block: &LightEthereumBlock, +) -> subgraph::TriggerData { + let mock_entity = create_mock_entity(block); + subgraph::TriggerData { + source: filter.subgraph.clone(), + entity: mock_entity, + entity_type: filter.entities.first().unwrap().clone(), + } +} + +fn create_mock_entity(block: &LightEthereumBlock) -> Entity { + let id = DeploymentHash::new("test").unwrap(); + let data_schema = InputSchema::parse_latest( + "type Block @entity { id: Bytes!, number: BigInt!, hash: Bytes! }", + id.clone(), + ) + .unwrap(); + let hash = Value::Bytes(scalar::Bytes::from(block.hash.unwrap().as_bytes().to_vec())); + let data = data_schema + .make_entity(vec![ + ("id".into(), hash.clone()), + ( + "number".into(), + Value::BigInt(scalar::BigInt::from(block.number())), + ), + ("hash".into(), hash), + ]) + .unwrap(); + + data +} + /// Returns blocks with triggers, corresponding to the specified range and filters; and the resolved /// `to` block, which is the nearest non-null block greater than or equal to the passed `to` block. /// If a block contains no triggers, there may be no corresponding item in the stream. @@ -1743,13 +1825,33 @@ pub(crate) async fn blocks_with_triggers( subgraph_metrics: Arc, from: BlockNumber, to: BlockNumber, - filter: &TriggerFilter, + filter: &Arc>, unified_api_version: UnifiedMappingApiVersion, ) -> Result<(Vec>, BlockNumber), Error> { // Each trigger filter needs to be queried for the same block range // and the blocks yielded need to be deduped. If any error occurs // while searching for a trigger type, the entire operation fails. let eth = adapter.clone(); + let subgraph_filter = filter.subgraph_filter.clone(); + + // TODO(krishna): In the initial implementation we do not allow any other datasource type + // When using subgraph data sources, there if subgraph_filter is not empty, we can return + // by just processing the subgraph triggers. + if !subgraph_filter.is_empty() { + return subgraph_triggers( + adapter.clone(), + logger.clone(), + chain_store.clone(), + subgraph_metrics.clone(), + from, + to, + filter, + unified_api_version, + ) + .await; + } + + let filter = filter.filter.clone(); let call_filter = EthereumCallFilter::from(&filter.block); // Scan the block range to find relevant triggers diff --git a/graph/src/blockchain/mod.rs b/graph/src/blockchain/mod.rs index e6c410acc3a..ec281f48ff6 100644 --- a/graph/src/blockchain/mod.rs +++ b/graph/src/blockchain/mod.rs @@ -256,7 +256,6 @@ pub struct TriggerFilterWrapper { pub subgraph_filter: Vec, } - #[derive(Clone, Debug)] pub struct SubgraphFilter { pub subgraph: DeploymentHash, @@ -265,10 +264,7 @@ pub struct SubgraphFilter { } impl TriggerFilterWrapper { - pub fn new( - filter: C::TriggerFilter, - subgraph_filter: Vec, - ) -> Self { + pub fn new(filter: C::TriggerFilter, subgraph_filter: Vec) -> Self { Self { filter: Arc::new(filter), subgraph_filter, From 91709422c7a7aa4da4c0b8287a1d2f2234659d68 Mon Sep 17 00:00:00 2001 From: incrypto32 Date: Wed, 24 Jul 2024 14:09:19 +0530 Subject: [PATCH 17/29] chain,graph : use `chain_head_ptr` method from adapter --- chain/arweave/src/chain.rs | 4 +++ chain/cosmos/src/chain.rs | 4 +++ chain/ethereum/src/chain.rs | 5 ++++ chain/near/src/chain.rs | 4 +++ chain/starknet/src/chain.rs | 4 +++ chain/substreams/src/trigger.rs | 4 +++ graph/src/blockchain/block_stream.rs | 31 +++++--------------- graph/src/blockchain/mock.rs | 5 ++++ graph/src/blockchain/polling_block_stream.rs | 2 +- tests/src/fixture/mod.rs | 4 +++ 10 files changed, 42 insertions(+), 25 deletions(-) diff --git a/chain/arweave/src/chain.rs b/chain/arweave/src/chain.rs index 163aed06204..d890279af43 100644 --- a/chain/arweave/src/chain.rs +++ b/chain/arweave/src/chain.rs @@ -203,6 +203,10 @@ impl TriggersAdapterTrait for TriggersAdapter { panic!("Should never be called since not used by FirehoseBlockStream") } + async fn chain_head_ptr(&self) -> Result, Error> { + unimplemented!() + } + async fn triggers_in_block( &self, logger: &Logger, diff --git a/chain/cosmos/src/chain.rs b/chain/cosmos/src/chain.rs index 7a20adc127f..b7f7d9d2fb5 100644 --- a/chain/cosmos/src/chain.rs +++ b/chain/cosmos/src/chain.rs @@ -197,6 +197,10 @@ impl TriggersAdapterTrait for TriggersAdapter { panic!("Should never be called since not used by FirehoseBlockStream") } + async fn chain_head_ptr(&self) -> Result, Error> { + unimplemented!() + } + async fn scan_triggers( &self, _from: BlockNumber, diff --git a/chain/ethereum/src/chain.rs b/chain/ethereum/src/chain.rs index b5a507565a7..2fff79ff7d2 100644 --- a/chain/ethereum/src/chain.rs +++ b/chain/ethereum/src/chain.rs @@ -776,6 +776,11 @@ impl TriggersAdapterTrait for TriggersAdapter { .await } + async fn chain_head_ptr(&self) -> Result, Error> { + let chain_store = self.chain_store.clone(); + chain_store.chain_head_ptr().await + } + async fn triggers_in_block( &self, logger: &Logger, diff --git a/chain/near/src/chain.rs b/chain/near/src/chain.rs index 73a17a6640a..18450159cf4 100644 --- a/chain/near/src/chain.rs +++ b/chain/near/src/chain.rs @@ -336,6 +336,10 @@ impl TriggersAdapterTrait for TriggersAdapter { panic!("Should never be called since not used by FirehoseBlockStream") } + async fn chain_head_ptr(&self) -> Result, Error> { + unimplemented!() + } + async fn triggers_in_block( &self, logger: &Logger, diff --git a/chain/starknet/src/chain.rs b/chain/starknet/src/chain.rs index 620c8412760..2d69e2a16c6 100644 --- a/chain/starknet/src/chain.rs +++ b/chain/starknet/src/chain.rs @@ -383,6 +383,10 @@ impl TriggersAdapterTrait for TriggersAdapter { panic!("Should never be called since FirehoseBlockStream cannot resolve it") } + async fn chain_head_ptr(&self) -> Result, Error> { + unimplemented!() + } + // Returns a sequence of blocks in increasing order of block number. // Each block will include all of its triggers that match the given `filter`. // The sequence may omit blocks that contain no triggers, diff --git a/chain/substreams/src/trigger.rs b/chain/substreams/src/trigger.rs index 5f919a12a90..becfecac505 100644 --- a/chain/substreams/src/trigger.rs +++ b/chain/substreams/src/trigger.rs @@ -137,6 +137,10 @@ impl blockchain::TriggersAdapter for TriggersAdapter { unimplemented!() } + async fn chain_head_ptr(&self) -> Result, Error> { + unimplemented!() + } + async fn scan_triggers( &self, _from: BlockNumber, diff --git a/graph/src/blockchain/block_stream.rs b/graph/src/blockchain/block_stream.rs index a528bc97c15..523a1e0a469 100644 --- a/graph/src/blockchain/block_stream.rs +++ b/graph/src/blockchain/block_stream.rs @@ -353,31 +353,11 @@ impl TriggersAdapter for TriggersAdapterWrapper { async fn parent_ptr(&self, block: &BlockPtr) -> Result, Error> { self.adapter.parent_ptr(block).await } -} -// fn create_mock_trigger() -> Trigger { -// let entity = create_mock_entity(); -// Trigger::Subgraph(subgraph::TriggerData { -// source: DeploymentHash::new("test").unwrap(), -// entity, -// entity_type: "User".to_string(), -// }) -// } - -// fn create_mock_entity() -> Entity { -// let schema = InputSchema::parse_latest( -// "type User @entity { id: String!, val: String! }", -// DeploymentHash::new("test").unwrap(), -// ) -// .unwrap(); - -// schema -// .make_entity(vec![ -// ("id".into(), Value::String("id".to_owned())), -// ("val".into(), Value::String("content".to_owned())), -// ]) -// .unwrap() -// } + async fn chain_head_ptr(&self) -> Result, Error> { + self.adapter.chain_head_ptr().await + } +} #[async_trait] pub trait TriggersAdapter: Send + Sync { @@ -419,6 +399,9 @@ pub trait TriggersAdapter: Send + Sync { /// Get pointer to parent of `block`. This is called when reverting `block`. async fn parent_ptr(&self, block: &BlockPtr) -> Result, Error>; + + /// Get pointer to parent of `block`. This is called when reverting `block`. + async fn chain_head_ptr(&self) -> Result, Error>; } #[async_trait] diff --git a/graph/src/blockchain/mock.rs b/graph/src/blockchain/mock.rs index c08f688a4b4..b1bef71fda9 100644 --- a/graph/src/blockchain/mock.rs +++ b/graph/src/blockchain/mock.rs @@ -228,6 +228,11 @@ impl TriggersAdapter for MockTriggersAdapter { todo!() } + async fn chain_head_ptr(&self) -> Result, Error> { + unimplemented!() + } + + async fn scan_triggers( &self, from: crate::components::store::BlockNumber, diff --git a/graph/src/blockchain/polling_block_stream.rs b/graph/src/blockchain/polling_block_stream.rs index 8cf466a6486..307f71cf283 100644 --- a/graph/src/blockchain/polling_block_stream.rs +++ b/graph/src/blockchain/polling_block_stream.rs @@ -218,7 +218,7 @@ where let max_block_range_size = self.max_block_range_size; // Get pointers from database for comparison - let head_ptr_opt = ctx.chain_store.chain_head_ptr().await?; + let head_ptr_opt = ctx.adapter.chain_head_ptr().await?; let subgraph_ptr = self.current_block.clone(); // If chain head ptr is not set yet diff --git a/tests/src/fixture/mod.rs b/tests/src/fixture/mod.rs index 358c5a8e425..3d329eb6163 100644 --- a/tests/src/fixture/mod.rs +++ b/tests/src/fixture/mod.rs @@ -1002,6 +1002,10 @@ impl TriggersAdapter for MockTriggersAdapter { todo!() } + async fn chain_head_ptr(&self) -> Result, Error> { + todo!() + } + async fn scan_triggers( &self, _from: BlockNumber, From da0c25411513982602015f2aea8f76687e3948f5 Mon Sep 17 00:00:00 2001 From: incrypto32 Date: Thu, 25 Jul 2024 10:11:26 +0530 Subject: [PATCH 18/29] chain, core, graph : use TriggersAdapterWrapper at top level --- chain/ethereum/src/chain.rs | 85 +++++--------------- chain/near/src/chain.rs | 15 +--- chain/starknet/src/chain.rs | 14 +--- chain/substreams/src/block_stream.rs | 14 +--- core/src/subgraph/inputs.rs | 4 +- core/src/subgraph/instance_manager.rs | 7 +- graph/src/blockchain/block_stream.rs | 29 ++++--- graph/src/blockchain/polling_block_stream.rs | 6 +- tests/src/fixture/mod.rs | 50 +++--------- 9 files changed, 69 insertions(+), 155 deletions(-) diff --git a/chain/ethereum/src/chain.rs b/chain/ethereum/src/chain.rs index 2fff79ff7d2..73489ae003f 100644 --- a/chain/ethereum/src/chain.rs +++ b/chain/ethereum/src/chain.rs @@ -133,62 +133,16 @@ impl BlockStreamBuilder for EthereumStreamBuilder { filter: Arc>, unified_api_version: UnifiedMappingApiVersion, ) -> Result>> { - let requirements = filter.filter.node_capabilities(); - let adapter = chain - .triggers_adapter(&deployment, &requirements, unified_api_version.clone()) - .unwrap_or_else(|_| { - panic!( - "no adapter for network {} with capabilities {}", - chain.name, requirements - ) - }); - - let adapter = Arc::new(TriggersAdapterWrapper::new(adapter, source_subgraph_stores)); - - let logger = chain - .logger_factory - .subgraph_logger(&deployment) - .new(o!("component" => "BlockStream")); - let chain_store = chain.chain_store(); - let chain_head_update_stream = chain - .chain_head_update_listener - .subscribe(chain.name.to_string(), logger.clone()); - - // Special case: Detect Celo and set the threshold to 0, so that eth_getLogs is always used. - // This is ok because Celo blocks are always final. And we _need_ to do this because - // some events appear only in eth_getLogs but not in transaction receipts. - // See also ca0edc58-0ec5-4c89-a7dd-2241797f5e50. - let chain_id = match chain.chain_client().as_ref() { - ChainClient::Rpc(adapter) => { - adapter - .cheapest() - .await - .ok_or(anyhow!("unable to get eth adapter for chan_id call"))? - .chain_id() - .await? - } - _ => panic!("expected rpc when using polling blockstream"), - }; - let reorg_threshold = match CELO_CHAIN_IDS.contains(&chain_id) { - false => chain.reorg_threshold, - true => 0, - }; - - Ok(Box::new(PollingBlockStream::new( - chain_store, - chain_head_update_stream, - adapter, - chain.node_id.clone(), - deployment.hash, - filter, + self.build_polling( + chain, + deployment, start_blocks, - reorg_threshold, - logger, - ENV_VARS.max_block_range_size, - ENV_VARS.target_triggers_per_block_range, - unified_api_version, + source_subgraph_stores, subgraph_current_block, - ))) + filter, + unified_api_version, + ) + .await } async fn build_polling( @@ -196,19 +150,23 @@ impl BlockStreamBuilder for EthereumStreamBuilder { chain: &Chain, deployment: DeploymentLocator, start_blocks: Vec, + source_subgraph_stores: Vec<(DeploymentHash, Arc)>, subgraph_current_block: Option, filter: Arc>, unified_api_version: UnifiedMappingApiVersion, ) -> Result>> { let requirements = filter.filter.node_capabilities(); - let adapter = chain - .triggers_adapter(&deployment, &requirements, unified_api_version.clone()) - .unwrap_or_else(|_| { - panic!( - "no adapter for network {} with capabilities {}", - chain.name, requirements - ) - }); + let adapter = TriggersAdapterWrapper::new( + chain + .triggers_adapter(&deployment, &requirements, unified_api_version.clone()) + .unwrap_or_else(|_| { + panic!( + "no adapter for network {} with capabilities {}", + chain.name, requirements + ) + }), + source_subgraph_stores, + ); let logger = chain .logger_factory @@ -242,7 +200,7 @@ impl BlockStreamBuilder for EthereumStreamBuilder { Ok(Box::new(PollingBlockStream::new( chain_store, chain_head_update_stream, - adapter, + Arc::new(adapter), chain.node_id.clone(), deployment.hash, filter, @@ -507,6 +465,7 @@ impl Blockchain for Chain { self, deployment, start_blocks, + source_subgraph_stores, current_ptr, filter, unified_api_version, diff --git a/chain/near/src/chain.rs b/chain/near/src/chain.rs index 18450159cf4..4d67e49bd48 100644 --- a/chain/near/src/chain.rs +++ b/chain/near/src/chain.rs @@ -108,20 +108,6 @@ impl BlockStreamBuilder for NearStreamBuilder { chain.metrics_registry.clone(), ))) } - - async fn build_subgraph_block_stream( - &self, - _chain: &Chain, - _deployment: DeploymentLocator, - _start_blocks: Vec, - _source_subgraph_stores: Vec<(DeploymentHash, Arc)>, - _subgraph_current_block: Option, - _filter: Arc>, - _unified_api_version: UnifiedMappingApiVersion, - ) -> Result>> { - unimplemented!() - } - async fn build_firehose( &self, chain: &Chain, @@ -164,6 +150,7 @@ impl BlockStreamBuilder for NearStreamBuilder { _chain: &Chain, _deployment: DeploymentLocator, _start_blocks: Vec, + _source_subgraph_stores: Vec<(DeploymentHash, Arc)>, _subgraph_current_block: Option, _filter: Arc>, _unified_api_version: UnifiedMappingApiVersion, diff --git a/chain/starknet/src/chain.rs b/chain/starknet/src/chain.rs index 2d69e2a16c6..583ea30bd69 100644 --- a/chain/starknet/src/chain.rs +++ b/chain/starknet/src/chain.rs @@ -197,19 +197,6 @@ impl BlockStreamBuilder for StarknetStreamBuilder { unimplemented!() } - async fn build_subgraph_block_stream( - &self, - _chain: &Chain, - _deployment: DeploymentLocator, - _start_blocks: Vec, - _source_subgraph_stores: Vec<(DeploymentHash, Arc)>, - _subgraph_current_block: Option, - _filter: Arc>, - _unified_api_version: UnifiedMappingApiVersion, - ) -> Result>> { - unimplemented!() - } - async fn build_firehose( &self, chain: &Chain, @@ -252,6 +239,7 @@ impl BlockStreamBuilder for StarknetStreamBuilder { _chain: &Chain, _deployment: DeploymentLocator, _start_blocks: Vec, + _source_subgraph_stores: Vec<(DeploymentHash, Arc)>, _subgraph_current_block: Option, _filter: Arc>, _unified_api_version: UnifiedMappingApiVersion, diff --git a/chain/substreams/src/block_stream.rs b/chain/substreams/src/block_stream.rs index 24cedade570..3687bab0b91 100644 --- a/chain/substreams/src/block_stream.rs +++ b/chain/substreams/src/block_stream.rs @@ -99,24 +99,12 @@ impl BlockStreamBuilderTrait for BlockStreamBuilder { unimplemented!() } - async fn build_subgraph_block_stream( - &self, - _chain: &Chain, - _deployment: DeploymentLocator, - _start_blocks: Vec, - _source_subgraph_stores: Vec<(DeploymentHash, Arc)>, - _subgraph_current_block: Option, - _filter: Arc>, - _unified_api_version: UnifiedMappingApiVersion, - ) -> Result>> { - unimplemented!() - } - async fn build_polling( &self, _chain: &Chain, _deployment: DeploymentLocator, _start_blocks: Vec, + _source_subgraph_stores: Vec<(DeploymentHash, Arc)>, _subgraph_current_block: Option, _filter: Arc>, _unified_api_version: UnifiedMappingApiVersion, diff --git a/core/src/subgraph/inputs.rs b/core/src/subgraph/inputs.rs index 3a12e33415f..386b1a5e35b 100644 --- a/core/src/subgraph/inputs.rs +++ b/core/src/subgraph/inputs.rs @@ -1,5 +1,5 @@ use graph::{ - blockchain::{Blockchain, TriggersAdapter}, + blockchain::{block_stream::TriggersAdapterWrapper, Blockchain}, components::{ store::{DeploymentLocator, SubgraphFork, WritableStore}, subgraph::ProofOfIndexingVersion, @@ -20,7 +20,7 @@ pub struct IndexingInputs { pub stop_block: Option, pub store: Arc, pub debug_fork: Option>, - pub triggers_adapter: Arc>, + pub triggers_adapter: Arc>, pub chain: Arc, pub templates: Arc>>, pub unified_api_version: UnifiedMappingApiVersion, diff --git a/core/src/subgraph/instance_manager.rs b/core/src/subgraph/instance_manager.rs index 11ce5f065c7..20ad464339f 100644 --- a/core/src/subgraph/instance_manager.rs +++ b/core/src/subgraph/instance_manager.rs @@ -6,7 +6,7 @@ use crate::subgraph::Decoder; use std::collections::BTreeSet; use crate::subgraph::runner::SubgraphRunner; -use graph::blockchain::block_stream::BlockStreamMetrics; +use graph::blockchain::block_stream::{BlockStreamMetrics, TriggersAdapterWrapper}; use graph::blockchain::{Blockchain, BlockchainKind, DataSource, NodeCapabilities}; use graph::components::metrics::gas::GasMetrics; use graph::components::store::WritableStore; @@ -501,6 +501,11 @@ impl SubgraphInstanceManager { ) .await?; + let triggers_adapter = Arc::new(TriggersAdapterWrapper::new( + triggers_adapter, + subgraph_data_source_writables.clone(), + )); + let inputs = IndexingInputs { deployment: deployment.clone(), features, diff --git a/graph/src/blockchain/block_stream.rs b/graph/src/blockchain/block_stream.rs index 523a1e0a469..c043a5d42eb 100644 --- a/graph/src/blockchain/block_stream.rs +++ b/graph/src/blockchain/block_stream.rs @@ -145,6 +145,7 @@ pub trait BlockStreamBuilder: Send + Sync { chain: &C, deployment: DeploymentLocator, start_blocks: Vec, + source_subgraph_stores: Vec<(DeploymentHash, Arc)>, subgraph_current_block: Option, filter: Arc>, unified_api_version: UnifiedMappingApiVersion, @@ -159,7 +160,18 @@ pub trait BlockStreamBuilder: Send + Sync { subgraph_current_block: Option, filter: Arc>, unified_api_version: UnifiedMappingApiVersion, - ) -> Result>>; + ) -> Result>> { + self.build_polling( + chain, + deployment, + start_blocks, + source_subgraph_stores, + subgraph_current_block, + filter, + unified_api_version, + ) + .await + } } #[derive(Debug, Clone)] @@ -316,9 +328,8 @@ impl TriggersAdapterWrapper { } } -#[async_trait] -impl TriggersAdapter for TriggersAdapterWrapper { - async fn ancestor_block( +impl TriggersAdapterWrapper { + pub async fn ancestor_block( &self, ptr: BlockPtr, offset: BlockNumber, @@ -328,7 +339,7 @@ impl TriggersAdapter for TriggersAdapterWrapper { } // TODO: Do a proper implementation, this is a complete mock implementation - async fn scan_triggers( + pub async fn scan_triggers( &self, from: BlockNumber, to: BlockNumber, @@ -337,7 +348,7 @@ impl TriggersAdapter for TriggersAdapterWrapper { self.adapter.scan_triggers(from, to, filter).await } - async fn triggers_in_block( + pub async fn triggers_in_block( &self, logger: &Logger, block: C::Block, @@ -346,15 +357,15 @@ impl TriggersAdapter for TriggersAdapterWrapper { self.adapter.triggers_in_block(logger, block, filter).await } - async fn is_on_main_chain(&self, ptr: BlockPtr) -> Result { + pub async fn is_on_main_chain(&self, ptr: BlockPtr) -> Result { self.adapter.is_on_main_chain(ptr).await } - async fn parent_ptr(&self, block: &BlockPtr) -> Result, Error> { + pub async fn parent_ptr(&self, block: &BlockPtr) -> Result, Error> { self.adapter.parent_ptr(block).await } - async fn chain_head_ptr(&self) -> Result, Error> { + pub async fn chain_head_ptr(&self) -> Result, Error> { self.adapter.chain_head_ptr().await } } diff --git a/graph/src/blockchain/polling_block_stream.rs b/graph/src/blockchain/polling_block_stream.rs index 307f71cf283..81fb4804c97 100644 --- a/graph/src/blockchain/polling_block_stream.rs +++ b/graph/src/blockchain/polling_block_stream.rs @@ -9,7 +9,7 @@ use std::time::Duration; use super::block_stream::{ BlockStream, BlockStreamError, BlockStreamEvent, BlockWithTriggers, ChainHeadUpdateStream, - FirehoseCursor, TriggersAdapter, BUFFERED_BLOCK_STREAM_SIZE, + FirehoseCursor, TriggersAdapterWrapper, BUFFERED_BLOCK_STREAM_SIZE, }; use super::{Block, BlockPtr, Blockchain, TriggerFilterWrapper}; @@ -79,7 +79,7 @@ where C: Blockchain, { chain_store: Arc, - adapter: Arc>, + adapter: Arc>, node_id: NodeId, subgraph_id: DeploymentHash, // This is not really a block number, but the (unsigned) difference @@ -146,7 +146,7 @@ where pub fn new( chain_store: Arc, chain_head_update_stream: ChainHeadUpdateStream, - adapter: Arc>, + adapter: Arc>, node_id: NodeId, subgraph_id: DeploymentHash, filter: Arc>, diff --git a/tests/src/fixture/mod.rs b/tests/src/fixture/mod.rs index 3d329eb6163..b58e436eba2 100644 --- a/tests/src/fixture/mod.rs +++ b/tests/src/fixture/mod.rs @@ -718,7 +718,7 @@ impl BlockStreamBuilder for MutexBlockStreamBuilder { unimplemented!(); } - async fn build_subgraph_block_stream( + async fn build_polling( &self, chain: &C, deployment: DeploymentLocator, @@ -731,7 +731,7 @@ impl BlockStreamBuilder for MutexBlockStreamBuilder { let builder = self.0.lock().unwrap().clone(); builder - .build_subgraph_block_stream( + .build_polling( chain, deployment, start_blocks, @@ -742,18 +742,6 @@ impl BlockStreamBuilder for MutexBlockStreamBuilder { ) .await } - - async fn build_polling( - &self, - _chain: &C, - _deployment: DeploymentLocator, - _start_blocks: Vec, - _subgraph_current_block: Option, - _filter: Arc>, - _unified_api_version: graph::data::subgraph::UnifiedMappingApiVersion, - ) -> anyhow::Result>> { - unimplemented!("only firehose mode should be used for tests") - } } /// `chain` is the sequence of chain heads to be processed. If the next block to be processed in the @@ -782,17 +770,17 @@ where unimplemented!() } - async fn build_subgraph_block_stream( + async fn build_firehose( &self, _chain: &C, _deployment: DeploymentLocator, - _start_blocks: Vec, - _source_subgraph_stores: Vec<(DeploymentHash, Arc)>, - subgraph_current_block: Option, - _filter: Arc>, + _block_cursor: FirehoseCursor, + _start_blocks: Vec, + current_block: Option, + _filter: Arc, _unified_api_version: graph::data::subgraph::UnifiedMappingApiVersion, ) -> anyhow::Result>> { - let current_idx = subgraph_current_block.map(|current_block| { + let current_idx = current_block.map(|current_block| { self.chain .iter() .enumerate() @@ -805,17 +793,17 @@ where })) } - async fn build_firehose( + async fn build_polling( &self, _chain: &C, _deployment: DeploymentLocator, - _block_cursor: FirehoseCursor, _start_blocks: Vec, - current_block: Option, - _filter: Arc, + _source_subgraph_stores: Vec<(DeploymentHash, Arc)>, + subgraph_current_block: Option, + _filter: Arc>, _unified_api_version: graph::data::subgraph::UnifiedMappingApiVersion, ) -> anyhow::Result>> { - let current_idx = current_block.map(|current_block| { + let current_idx = subgraph_current_block.map(|current_block| { self.chain .iter() .enumerate() @@ -827,18 +815,6 @@ where stream: Box::pin(stream_events(self.chain.clone(), current_idx)), })) } - - async fn build_polling( - &self, - _chain: &C, - _deployment: DeploymentLocator, - _start_blocks: Vec, - _subgraph_current_block: Option, - _filter: Arc>, - _unified_api_version: graph::data::subgraph::UnifiedMappingApiVersion, - ) -> anyhow::Result>> { - unimplemented!("only firehose mode should be used for tests") - } } struct StaticStream { From 5fd12194e59a9c5ce564de3e8f8d189f7bc118a3 Mon Sep 17 00:00:00 2001 From: incrypto32 Date: Thu, 25 Jul 2024 13:35:51 +0530 Subject: [PATCH 19/29] chain, graph : move subgraph trigger scanning back to TriggersAdapterWrapper --- chain/arweave/src/chain.rs | 14 ++- chain/cosmos/src/chain.rs | 15 ++- chain/ethereum/src/adapter.rs | 6 +- chain/ethereum/src/chain.rs | 32 +++++- chain/ethereum/src/ethereum_adapter.rs | 106 +----------------- chain/near/src/chain.rs | 16 ++- chain/starknet/src/chain.rs | 15 ++- chain/substreams/src/chain.rs | 2 +- chain/substreams/src/trigger.rs | 15 ++- graph/src/blockchain/block_stream.rs | 90 ++++++++++++++- graph/src/blockchain/mock.rs | 16 ++- graph/src/blockchain/mod.rs | 6 +- graph/src/blockchain/polling_block_stream.rs | 2 +- .../subgraph/proof_of_indexing/online.rs | 4 +- tests/src/fixture/mod.rs | 12 +- tests/tests/runner_tests.rs | 23 +++- 16 files changed, 228 insertions(+), 146 deletions(-) diff --git a/chain/arweave/src/chain.rs b/chain/arweave/src/chain.rs index d890279af43..e897e10d8d8 100644 --- a/chain/arweave/src/chain.rs +++ b/chain/arweave/src/chain.rs @@ -27,11 +27,13 @@ use graph::{ prelude::{async_trait, o, BlockNumber, ChainStore, Error, Logger, LoggerFactory}, }; use prost::Message; +use std::collections::HashSet; use std::sync::Arc; use crate::adapter::TriggerFilter; use crate::data_source::{DataSourceTemplate, UnresolvedDataSourceTemplate}; use crate::trigger::{self, ArweaveTrigger}; +use crate::Block as ArweaveBlock; use crate::{ codec, data_source::{DataSource, UnresolvedDataSource}, @@ -138,7 +140,7 @@ impl Blockchain for Chain { let firehose_mapper = Arc::new(FirehoseMapper { adapter, - filter: filter.filter.clone(), + filter: filter.chain_filter.clone(), }); Ok(Box::new(FirehoseBlockStream::new( @@ -198,7 +200,7 @@ impl TriggersAdapterTrait for TriggersAdapter { &self, _from: BlockNumber, _to: BlockNumber, - _filter: &Arc>, + _filter: &TriggerFilter, ) -> Result<(Vec>, BlockNumber), Error> { panic!("Should never be called since not used by FirehoseBlockStream") } @@ -266,6 +268,14 @@ impl TriggersAdapterTrait for TriggersAdapter { number: block.number.saturating_sub(1), })) } + + async fn load_blocks_by_numbers( + &self, + _logger: Logger, + _block_numbers: HashSet, + ) -> Result, Error> { + todo!() + } } pub struct FirehoseMapper { diff --git a/chain/cosmos/src/chain.rs b/chain/cosmos/src/chain.rs index b7f7d9d2fb5..83a299b6163 100644 --- a/chain/cosmos/src/chain.rs +++ b/chain/cosmos/src/chain.rs @@ -4,6 +4,7 @@ use graph::components::adapter::ChainId; use graph::env::EnvVars; use graph::prelude::{DeploymentHash, MetricsRegistry}; use graph::substreams::Clock; +use std::collections::HashSet; use std::convert::TryFrom; use std::sync::Arc; @@ -33,7 +34,7 @@ use crate::data_source::{ DataSource, DataSourceTemplate, EventOrigin, UnresolvedDataSource, UnresolvedDataSourceTemplate, }; use crate::trigger::CosmosTrigger; -use crate::{codec, TriggerFilter}; +use crate::{codec, Block, TriggerFilter}; pub struct Chain { logger_factory: LoggerFactory, @@ -132,7 +133,7 @@ impl Blockchain for Chain { let firehose_mapper = Arc::new(FirehoseMapper { adapter, - filter: filter.filter.clone(), + filter: filter.chain_filter.clone(), }); Ok(Box::new(FirehoseBlockStream::new( @@ -197,6 +198,14 @@ impl TriggersAdapterTrait for TriggersAdapter { panic!("Should never be called since not used by FirehoseBlockStream") } + async fn load_blocks_by_numbers( + &self, + _logger: Logger, + _block_numbers: HashSet, + ) -> Result, Error> { + unimplemented!() + } + async fn chain_head_ptr(&self) -> Result, Error> { unimplemented!() } @@ -205,7 +214,7 @@ impl TriggersAdapterTrait for TriggersAdapter { &self, _from: BlockNumber, _to: BlockNumber, - _filter: &Arc>, + _filter: &TriggerFilter, ) -> Result<(Vec>, BlockNumber), Error> { panic!("Should never be called since not used by FirehoseBlockStream") } diff --git a/chain/ethereum/src/adapter.rs b/chain/ethereum/src/adapter.rs index 31083724809..3d4dc00c030 100644 --- a/chain/ethereum/src/adapter.rs +++ b/chain/ethereum/src/adapter.rs @@ -1111,9 +1111,9 @@ pub trait EthereumAdapter: Send + Sync + 'static { async fn load_blocks_by_numbers( &self, - logger: Logger, - chain_store: Arc, - block_numbers: HashSet, + _logger: Logger, + _chain_store: Arc, + _block_numbers: HashSet, ) -> Box, Error = Error> + Send>; /// Load Ethereum blocks in bulk, returning results as they come back as a Stream. diff --git a/chain/ethereum/src/chain.rs b/chain/ethereum/src/chain.rs index 73489ae003f..9b8e71b0fe2 100644 --- a/chain/ethereum/src/chain.rs +++ b/chain/ethereum/src/chain.rs @@ -155,7 +155,7 @@ impl BlockStreamBuilder for EthereumStreamBuilder { filter: Arc>, unified_api_version: UnifiedMappingApiVersion, ) -> Result>> { - let requirements = filter.filter.node_capabilities(); + let requirements = filter.chain_filter.node_capabilities(); let adapter = TriggersAdapterWrapper::new( chain .triggers_adapter(&deployment, &requirements, unified_api_version.clone()) @@ -480,7 +480,7 @@ impl Blockchain for Chain { store.firehose_cursor(), start_blocks, current_ptr, - filter.filter.clone(), + filter.chain_filter.clone(), unified_api_version, ) .await @@ -717,7 +717,7 @@ impl TriggersAdapterTrait for TriggersAdapter { &self, from: BlockNumber, to: BlockNumber, - filter: &Arc>, + filter: &TriggerFilter, ) -> Result<(Vec>, BlockNumber), Error> { blocks_with_triggers( self.chain_client @@ -735,6 +735,30 @@ impl TriggersAdapterTrait for TriggersAdapter { .await } + async fn load_blocks_by_numbers( + &self, + logger: Logger, + block_numbers: HashSet, + ) -> Result> { + use graph::futures01::stream::Stream; + + let adapter = self + .chain_client + .rpc()? + .cheapest_with(&self.capabilities) + .await?; + + let blocks = adapter + .load_blocks_by_numbers(logger, self.chain_store.clone(), block_numbers) + .await + .map(|block| BlockFinality::Final(block)) + .collect() + .compat() + .await?; + + Ok(blocks) + } + async fn chain_head_ptr(&self) -> Result, Error> { let chain_store = self.chain_store.clone(); chain_store.chain_head_ptr().await @@ -771,7 +795,7 @@ impl TriggersAdapterTrait for TriggersAdapter { self.ethrpc_metrics.clone(), block_number, block_number, - &Arc::new(TriggerFilterWrapper::::new(filter.clone(), vec![])), // TODO(krishna): This is temporary until we take TriggerFilterWrapper as param in triggers_in_block + filter, self.unified_api_version.clone(), ) .await?; diff --git a/chain/ethereum/src/ethereum_adapter.rs b/chain/ethereum/src/ethereum_adapter.rs index 7e9786a6883..9fe0b8262b2 100644 --- a/chain/ethereum/src/ethereum_adapter.rs +++ b/chain/ethereum/src/ethereum_adapter.rs @@ -2,14 +2,12 @@ use futures03::{future::BoxFuture, stream::FuturesUnordered}; use graph::blockchain::client::ChainClient; use graph::blockchain::BlockHash; use graph::blockchain::ChainIdentifier; -use graph::blockchain::SubgraphFilter; -use graph::blockchain::TriggerFilterWrapper; + use graph::components::transaction_receipt::LightTransactionReceipt; use graph::data::store::ethereum::call; use graph::data::store::scalar; use graph::data::subgraph::UnifiedMappingApiVersion; use graph::data::subgraph::API_VERSION_0_0_7; -use graph::data_source::subgraph; use graph::futures01::stream; use graph::futures01::Future; use graph::futures01::Stream; @@ -21,10 +19,6 @@ use graph::prelude::ethabi::ParamType; use graph::prelude::ethabi::Token; use graph::prelude::tokio::try_join; use graph::prelude::web3::types::U256; -use graph::prelude::DeploymentHash; -use graph::prelude::Entity; -use graph::prelude::Value; -use graph::schema::InputSchema; use graph::slog::o; use graph::tokio::sync::RwLock; use graph::tokio::time::timeout; @@ -65,6 +59,7 @@ use crate::chain::BlockFinality; use crate::trigger::LogRef; use crate::Chain; use crate::NodeCapabilities; +use crate::TriggerFilter; use crate::{ adapter::{ ContractCall, ContractCallError, EthGetLogsFilter, EthereumAdapter as EthereumAdapterTrait, @@ -1729,81 +1724,6 @@ impl EthereumAdapterTrait for EthereumAdapter { } } -// TODO(krishna): Currently this is a mock implementation of subgraph triggers. -// This will be replaced with the actual implementation which will use the filters to -// query the database of the source subgraph and return the entity triggers. -async fn subgraph_triggers( - adapter: Arc, - logger: Logger, - chain_store: Arc, - _subgraph_metrics: Arc, - from: BlockNumber, - to: BlockNumber, - filter: &Arc>, - _unified_api_version: UnifiedMappingApiVersion, -) -> Result<(Vec>, BlockNumber), Error> { - let logger2 = logger.cheap_clone(); - let eth = adapter.clone(); - let to_ptr = eth.next_existing_ptr_to_number(&logger, to).await?; - let to = to_ptr.block_number(); - - let first_filter = filter.subgraph_filter.first().unwrap(); - - let blocks = adapter - .load_blocks_by_numbers( - logger.cheap_clone(), - chain_store.clone(), - HashSet::from_iter(from..=to), - ) - .await - .and_then(move |block| { - Ok(BlockWithTriggers::::new_with_subgraph_triggers( - BlockFinality::Final(block.clone()), - vec![create_mock_subgraph_trigger(first_filter, &block)], - &logger2, - )) - }) - .collect() - .compat() - .await?; - - Ok((blocks, to)) -} - -fn create_mock_subgraph_trigger( - filter: &SubgraphFilter, - block: &LightEthereumBlock, -) -> subgraph::TriggerData { - let mock_entity = create_mock_entity(block); - subgraph::TriggerData { - source: filter.subgraph.clone(), - entity: mock_entity, - entity_type: filter.entities.first().unwrap().clone(), - } -} - -fn create_mock_entity(block: &LightEthereumBlock) -> Entity { - let id = DeploymentHash::new("test").unwrap(); - let data_schema = InputSchema::parse_latest( - "type Block @entity { id: Bytes!, number: BigInt!, hash: Bytes! }", - id.clone(), - ) - .unwrap(); - let hash = Value::Bytes(scalar::Bytes::from(block.hash.unwrap().as_bytes().to_vec())); - let data = data_schema - .make_entity(vec![ - ("id".into(), hash.clone()), - ( - "number".into(), - Value::BigInt(scalar::BigInt::from(block.number())), - ), - ("hash".into(), hash), - ]) - .unwrap(); - - data -} - /// Returns blocks with triggers, corresponding to the specified range and filters; and the resolved /// `to` block, which is the nearest non-null block greater than or equal to the passed `to` block. /// If a block contains no triggers, there may be no corresponding item in the stream. @@ -1825,33 +1745,13 @@ pub(crate) async fn blocks_with_triggers( subgraph_metrics: Arc, from: BlockNumber, to: BlockNumber, - filter: &Arc>, + filter: &TriggerFilter, unified_api_version: UnifiedMappingApiVersion, ) -> Result<(Vec>, BlockNumber), Error> { // Each trigger filter needs to be queried for the same block range // and the blocks yielded need to be deduped. If any error occurs // while searching for a trigger type, the entire operation fails. let eth = adapter.clone(); - let subgraph_filter = filter.subgraph_filter.clone(); - - // TODO(krishna): In the initial implementation we do not allow any other datasource type - // When using subgraph data sources, there if subgraph_filter is not empty, we can return - // by just processing the subgraph triggers. - if !subgraph_filter.is_empty() { - return subgraph_triggers( - adapter.clone(), - logger.clone(), - chain_store.clone(), - subgraph_metrics.clone(), - from, - to, - filter, - unified_api_version, - ) - .await; - } - - let filter = filter.filter.clone(); let call_filter = EthereumCallFilter::from(&filter.block); // Scan the block range to find relevant triggers diff --git a/chain/near/src/chain.rs b/chain/near/src/chain.rs index 4d67e49bd48..44c0449adce 100644 --- a/chain/near/src/chain.rs +++ b/chain/near/src/chain.rs @@ -32,10 +32,12 @@ use graph::{ prelude::{async_trait, o, BlockNumber, ChainStore, Error, Logger, LoggerFactory}, }; use prost::Message; +use std::collections::HashSet; use std::sync::Arc; use crate::adapter::TriggerFilter; use crate::codec::substreams_triggers::BlockAndReceipts; +use crate::codec::Block; use crate::data_source::{DataSourceTemplate, UnresolvedDataSourceTemplate}; use crate::trigger::{self, NearTrigger}; use crate::{ @@ -243,7 +245,7 @@ impl Blockchain for Chain { deployment, store.firehose_cursor(), store.block_ptr(), - filter.filter.clone(), + filter.chain_filter.clone(), ) .await; } @@ -255,7 +257,7 @@ impl Blockchain for Chain { store.firehose_cursor(), start_blocks, store.block_ptr(), - filter.filter.clone(), + filter.chain_filter.clone(), unified_api_version, ) .await @@ -318,11 +320,19 @@ impl TriggersAdapterTrait for TriggersAdapter { &self, _from: BlockNumber, _to: BlockNumber, - _filter: &Arc>, + _filter: &TriggerFilter, ) -> Result<(Vec>, BlockNumber), Error> { panic!("Should never be called since not used by FirehoseBlockStream") } + async fn load_blocks_by_numbers( + &self, + _logger: Logger, + _block_numbers: HashSet, + ) -> Result> { + unimplemented!() + } + async fn chain_head_ptr(&self) -> Result, Error> { unimplemented!() } diff --git a/chain/starknet/src/chain.rs b/chain/starknet/src/chain.rs index 583ea30bd69..b6ff51a9bcd 100644 --- a/chain/starknet/src/chain.rs +++ b/chain/starknet/src/chain.rs @@ -30,7 +30,7 @@ use graph::{ slog::o, }; use prost::Message; -use std::sync::Arc; +use std::{collections::HashSet, sync::Arc}; use crate::{ adapter::TriggerFilter, @@ -39,6 +39,7 @@ use crate::{ DataSource, DataSourceTemplate, UnresolvedDataSource, UnresolvedDataSourceTemplate, }, trigger::{StarknetBlockTrigger, StarknetEventTrigger, StarknetTrigger}, + Block as StarknetBlock, }; pub struct Chain { @@ -126,7 +127,7 @@ impl Blockchain for Chain { store.firehose_cursor(), start_blocks, store.block_ptr(), - filter.filter.clone(), + filter.chain_filter.clone(), unified_api_version, ) .await @@ -371,6 +372,14 @@ impl TriggersAdapterTrait for TriggersAdapter { panic!("Should never be called since FirehoseBlockStream cannot resolve it") } + async fn load_blocks_by_numbers( + &self, + _logger: Logger, + _block_numbers: HashSet, + ) -> Result, Error> { + unimplemented!() + } + async fn chain_head_ptr(&self) -> Result, Error> { unimplemented!() } @@ -385,7 +394,7 @@ impl TriggersAdapterTrait for TriggersAdapter { &self, _from: BlockNumber, _to: BlockNumber, - _filter: &Arc>, + _filter: &TriggerFilter, ) -> Result<(Vec>, BlockNumber), Error> { panic!("Should never be called since not used by FirehoseBlockStream") } diff --git a/chain/substreams/src/chain.rs b/chain/substreams/src/chain.rs index b3cf8cca8a6..38ef49bdb5d 100644 --- a/chain/substreams/src/chain.rs +++ b/chain/substreams/src/chain.rs @@ -153,7 +153,7 @@ impl Blockchain for Chain { deployment, store.firehose_cursor(), store.block_ptr(), - filter.filter.clone(), + filter.chain_filter.clone(), ) .await } diff --git a/chain/substreams/src/trigger.rs b/chain/substreams/src/trigger.rs index becfecac505..db4034cd55c 100644 --- a/chain/substreams/src/trigger.rs +++ b/chain/substreams/src/trigger.rs @@ -1,8 +1,7 @@ use anyhow::Error; use graph::{ blockchain::{ - self, block_stream::BlockWithTriggers, BlockPtr, EmptyNodeCapabilities, - MappingTriggerTrait, TriggerFilterWrapper, + self, block_stream::BlockWithTriggers, BlockPtr, EmptyNodeCapabilities, MappingTriggerTrait, }, components::{ store::{DeploymentLocator, SubgraphFork}, @@ -17,7 +16,7 @@ use graph::{ }; use graph_runtime_wasm::module::ToAscPtr; use lazy_static::__Deref; -use std::sync::Arc; +use std::{collections::HashSet, sync::Arc}; use crate::{Block, Chain, NoopDataSourceTemplate, ParsedChanges}; @@ -137,6 +136,14 @@ impl blockchain::TriggersAdapter for TriggersAdapter { unimplemented!() } + async fn load_blocks_by_numbers( + &self, + _logger: Logger, + _block_numbers: HashSet, + ) -> Result, Error> { + unimplemented!() + } + async fn chain_head_ptr(&self) -> Result, Error> { unimplemented!() } @@ -145,7 +152,7 @@ impl blockchain::TriggersAdapter for TriggersAdapter { &self, _from: BlockNumber, _to: BlockNumber, - _filter: &Arc>, + _filter: &TriggerFilter, ) -> Result<(Vec>, BlockNumber), Error> { unimplemented!() } diff --git a/graph/src/blockchain/block_stream.rs b/graph/src/blockchain/block_stream.rs index c043a5d42eb..3c5f08851f2 100644 --- a/graph/src/blockchain/block_stream.rs +++ b/graph/src/blockchain/block_stream.rs @@ -1,3 +1,4 @@ +use crate::data::store::scalar; use crate::data_source::subgraph; use crate::substreams::Clock; use crate::substreams_rpc::response::Message as SubstreamsMessage; @@ -6,6 +7,7 @@ use anyhow::Error; use async_stream::stream; use futures03::Stream; use prost_types::Any; +use std::collections::HashSet; use std::fmt; use std::sync::Arc; use std::time::Instant; @@ -13,7 +15,9 @@ use thiserror::Error; use tokio::sync::mpsc::{self, Receiver, Sender}; use super::substreams_block_stream::SubstreamsLogData; -use super::{Block, BlockPtr, BlockTime, Blockchain, Trigger, TriggerFilterWrapper}; +use super::{ + Block, BlockPtr, BlockTime, Blockchain, SubgraphFilter, Trigger, TriggerFilterWrapper, +}; use crate::anyhow::Result; use crate::components::store::{BlockNumber, DeploymentLocator, WritableStore}; use crate::data::subgraph::UnifiedMappingApiVersion; @@ -345,7 +349,15 @@ impl TriggersAdapterWrapper { to: BlockNumber, filter: &Arc>, ) -> Result<(Vec>, BlockNumber), Error> { - self.adapter.scan_triggers(from, to, filter).await + if !filter.subgraph_filter.is_empty() { + return self + .subgraph_triggers(Logger::root(slog::Discard, o!()), from, to, filter) + .await; + } + + self.adapter + .scan_triggers(from, to, &filter.chain_filter) + .await } pub async fn triggers_in_block( @@ -368,6 +380,72 @@ impl TriggersAdapterWrapper { pub async fn chain_head_ptr(&self) -> Result, Error> { self.adapter.chain_head_ptr().await } + + // TODO(krishna): Currently this is a mock implementation of subgraph triggers. + // This will be replaced with the actual implementation which will use the filters to + // query the database of the source subgraph and return the entity triggers. + async fn subgraph_triggers( + &self, + logger: Logger, + from: BlockNumber, + to: BlockNumber, + filter: &Arc>, + ) -> Result<(Vec>, BlockNumber), Error> { + let logger2 = logger.cheap_clone(); + let adapter = self.adapter.clone(); + // let to_ptr = eth.next_existing_ptr_to_number(&logger, to).await?; + // let to = to_ptr.block_number(); + + let first_filter = filter.subgraph_filter.first().unwrap(); + + let blocks = adapter + .load_blocks_by_numbers(logger, HashSet::from_iter(from..=to)) + .await? + .into_iter() + .map(|block| { + let trigger_data = vec![Self::create_mock_subgraph_trigger(first_filter, &block)]; + BlockWithTriggers::new_with_subgraph_triggers(block, trigger_data, &logger2) + }) + .collect(); + + Ok((blocks, to)) + } + + fn create_mock_subgraph_trigger( + filter: &SubgraphFilter, + block: &C::Block, + ) -> subgraph::TriggerData { + let mock_entity = Self::create_mock_entity(block); + subgraph::TriggerData { + source: filter.subgraph.clone(), + entity: mock_entity, + entity_type: filter.entities.first().unwrap().clone(), + } + } + + fn create_mock_entity(block: &C::Block) -> Entity { + let id = DeploymentHash::new("test").unwrap(); + let data_schema = InputSchema::parse_latest( + "type Block @entity { id: Bytes!, number: BigInt!, hash: Bytes! }", + id.clone(), + ) + .unwrap(); + + let block = block.ptr(); + let hash = Value::Bytes(scalar::Bytes::from(block.hash_slice().to_vec())); + let data = data_schema + .make_entity(vec![ + ("id".into(), hash.clone()), + ( + "number".into(), + Value::BigInt(scalar::BigInt::from(block.block_number())), + ), + ("hash".into(), hash), + ]) + .unwrap(); + + data + } } #[async_trait] @@ -393,7 +471,7 @@ pub trait TriggersAdapter: Send + Sync { &self, from: BlockNumber, to: BlockNumber, - filter: &Arc>, + filter: &C::TriggerFilter, ) -> Result<(Vec>, BlockNumber), Error>; // Used for reprocessing blocks when creating a data source. @@ -413,6 +491,12 @@ pub trait TriggersAdapter: Send + Sync { /// Get pointer to parent of `block`. This is called when reverting `block`. async fn chain_head_ptr(&self) -> Result, Error>; + + async fn load_blocks_by_numbers( + &self, + logger: Logger, + block_numbers: HashSet, + ) -> Result>; } #[async_trait] diff --git a/graph/src/blockchain/mock.rs b/graph/src/blockchain/mock.rs index b1bef71fda9..21042921cb6 100644 --- a/graph/src/blockchain/mock.rs +++ b/graph/src/blockchain/mock.rs @@ -8,9 +8,10 @@ use crate::{ data::subgraph::UnifiedMappingApiVersion, prelude::{BlockHash, DataSourceTemplateInfo, DeploymentHash}, }; -use anyhow::Error; +use anyhow::{Error, Result}; use async_trait::async_trait; use serde::Deserialize; +use slog::Logger; use std::{collections::HashSet, convert::TryFrom, sync::Arc}; use super::{ @@ -228,16 +229,23 @@ impl TriggersAdapter for MockTriggersAdapter { todo!() } + async fn load_blocks_by_numbers( + &self, + _logger: Logger, + _block_numbers: HashSet, + ) -> Result> { + unimplemented!() + } + async fn chain_head_ptr(&self) -> Result, Error> { unimplemented!() } - async fn scan_triggers( &self, from: crate::components::store::BlockNumber, to: crate::components::store::BlockNumber, - filter: &Arc>, + filter: &MockTriggerFilter, ) -> Result< ( Vec>, @@ -269,7 +277,7 @@ impl TriggersAdapter for MockTriggersAdapter { async fn blocks_with_triggers( _from: crate::components::store::BlockNumber, to: crate::components::store::BlockNumber, - _filter: &Arc>, + _filter: &MockTriggerFilter, ) -> Result< ( Vec>, diff --git a/graph/src/blockchain/mod.rs b/graph/src/blockchain/mod.rs index ec281f48ff6..d1e07d2d36e 100644 --- a/graph/src/blockchain/mod.rs +++ b/graph/src/blockchain/mod.rs @@ -252,7 +252,7 @@ impl From for IngestorError { #[derive(Debug)] pub struct TriggerFilterWrapper { - pub filter: Arc, + pub chain_filter: Arc, pub subgraph_filter: Vec, } @@ -266,7 +266,7 @@ pub struct SubgraphFilter { impl TriggerFilterWrapper { pub fn new(filter: C::TriggerFilter, subgraph_filter: Vec) -> Self { Self { - filter: Arc::new(filter), + chain_filter: Arc::new(filter), subgraph_filter, } } @@ -275,7 +275,7 @@ impl TriggerFilterWrapper { impl Clone for TriggerFilterWrapper { fn clone(&self) -> Self { Self { - filter: self.filter.cheap_clone(), + chain_filter: self.chain_filter.cheap_clone(), subgraph_filter: self.subgraph_filter.clone(), } } diff --git a/graph/src/blockchain/polling_block_stream.rs b/graph/src/blockchain/polling_block_stream.rs index 81fb4804c97..5b37cd303b4 100644 --- a/graph/src/blockchain/polling_block_stream.rs +++ b/graph/src/blockchain/polling_block_stream.rs @@ -472,7 +472,7 @@ where .triggers_in_block( &self.logger, head_ancestor, - &self.filter.filter.clone(), + &self.filter.chain_filter.clone(), ) .await?; Ok(ReconciliationStep::ProcessDescendantBlocks(vec![block], 1)) diff --git a/graph/src/components/subgraph/proof_of_indexing/online.rs b/graph/src/components/subgraph/proof_of_indexing/online.rs index caaa76f0a76..f90fac969cf 100644 --- a/graph/src/components/subgraph/proof_of_indexing/online.rs +++ b/graph/src/components/subgraph/proof_of_indexing/online.rs @@ -146,8 +146,8 @@ impl BlockEventStream { fn write(&mut self, event: &ProofOfIndexingEvent<'_>) { let children = &[ 1, // kvp -> v - 0, // PoICausalityRegion.blocks: Vec - self.block_index, // Vec -> [i] + 0, // PoICausalityRegion.blocks: Result> + self.block_index, // Result> -> [i] 0, // Block.events -> Vec self.vec_length, ]; diff --git a/tests/src/fixture/mod.rs b/tests/src/fixture/mod.rs index b58e436eba2..b01eb4c7670 100644 --- a/tests/src/fixture/mod.rs +++ b/tests/src/fixture/mod.rs @@ -1,7 +1,7 @@ pub mod ethereum; pub mod substreams; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::marker::PhantomData; use std::sync::Mutex; use std::time::{Duration, Instant}; @@ -978,6 +978,14 @@ impl TriggersAdapter for MockTriggersAdapter { todo!() } + async fn load_blocks_by_numbers( + &self, + _logger: Logger, + _block_numbers: HashSet, + ) -> Result, Error> { + unimplemented!() + } + async fn chain_head_ptr(&self) -> Result, Error> { todo!() } @@ -986,7 +994,7 @@ impl TriggersAdapter for MockTriggersAdapter { &self, _from: BlockNumber, _to: BlockNumber, - _filter: &Arc>, + _filter: &C::TriggerFilter, ) -> Result<(Vec>, BlockNumber), Error> { todo!() } diff --git a/tests/tests/runner_tests.rs b/tests/tests/runner_tests.rs index eb409f6417c..8f01e4a98f2 100644 --- a/tests/tests/runner_tests.rs +++ b/tests/tests/runner_tests.rs @@ -501,10 +501,19 @@ async fn substreams_trigger_filter_construction() -> anyhow::Result<()> { let runner = ctx.runner_substreams(test_ptr(0)).await; let filter = runner.build_filter_for_test(); - assert_eq!(filter.filter.module_name(), "graph_out"); - assert_eq!(filter.filter.modules().as_ref().unwrap().modules.len(), 2); - assert_eq!(filter.filter.start_block().unwrap(), 0); - assert_eq!(filter.filter.data_sources_len(), 1); + assert_eq!(filter.chain_filter.module_name(), "graph_out"); + assert_eq!( + filter + .chain_filter + .modules() + .as_ref() + .unwrap() + .modules + .len(), + 2 + ); + assert_eq!(filter.chain_filter.start_block().unwrap(), 0); + assert_eq!(filter.chain_filter.data_sources_len(), 1); Ok(()) } @@ -526,7 +535,11 @@ async fn end_block() -> anyhow::Result<()> { let runner = ctx.runner(block_ptr.clone()).await; let runner = runner.run_for_test(false).await.unwrap(); let filter = runner.context().filter.as_ref().unwrap(); - let addresses = filter.filter.log().contract_addresses().collect::>(); + let addresses = filter + .chain_filter + .log() + .contract_addresses() + .collect::>(); if should_contain_addr { assert!(addresses.contains(&addr)); From 7b97a167cc262575faa24ebfac4979ffeb9a8654 Mon Sep 17 00:00:00 2001 From: Zoran Cvetkov Date: Wed, 17 Jul 2024 18:51:26 +0300 Subject: [PATCH 20/29] first attempt at reading entities for mutable ones --- graph/src/components/store/mod.rs | 9 ++ graph/src/components/store/traits.rs | 16 ++++ store/postgres/src/block_range.rs | 77 +++++++++++++++- store/postgres/src/deployment_store.rs | 13 ++- store/postgres/src/relational.rs | 24 ++++- store/postgres/src/relational_queries.rs | 92 +++++++++++++++++++- store/postgres/src/writable.rs | 41 ++++++++- store/test-store/tests/graph/entity_cache.rs | 9 ++ store/test-store/tests/postgres/writable.rs | 34 ++++++-- 9 files changed, 305 insertions(+), 10 deletions(-) diff --git a/graph/src/components/store/mod.rs b/graph/src/components/store/mod.rs index 31b0e62cfae..edf3f575512 100644 --- a/graph/src/components/store/mod.rs +++ b/graph/src/components/store/mod.rs @@ -21,6 +21,7 @@ use std::collections::btree_map::Entry; use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::fmt; use std::fmt::Display; +use std::ops::Range; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, RwLock}; use std::time::Duration; @@ -1038,6 +1039,14 @@ impl ReadStore for EmptyStore { Ok(BTreeMap::new()) } + fn get_range( + &self, + _key: &EntityKey, + _block_range: Range, + ) -> Result, StoreError> { + Ok(BTreeMap::new()) + } + fn get_derived( &self, _query: &DerivedEntityQuery, diff --git a/graph/src/components/store/traits.rs b/graph/src/components/store/traits.rs index 69ed67c16b2..6cc8ff23ec1 100644 --- a/graph/src/components/store/traits.rs +++ b/graph/src/components/store/traits.rs @@ -1,4 +1,5 @@ use std::collections::HashMap; +use std::ops::Range; use anyhow::Error; use async_trait::async_trait; @@ -227,6 +228,13 @@ pub trait ReadStore: Send + Sync + 'static { keys: BTreeSet, ) -> Result, StoreError>; + /// Looks up entities using the given store key for a range of blocks. + fn get_range( + &self, + key: &EntityKey, + block_range: Range, + ) -> Result, StoreError>; + /// Reverse lookup fn get_derived( &self, @@ -249,6 +257,14 @@ impl ReadStore for Arc { (**self).get_many(keys) } + fn get_range( + &self, + key: &EntityKey, + block_range: Range, + ) -> Result, StoreError> { + (**self).get_range(key, block_range) + } + fn get_derived( &self, entity_derived: &DerivedEntityQuery, diff --git a/store/postgres/src/block_range.rs b/store/postgres/src/block_range.rs index 1d81eac5e81..b6d11811917 100644 --- a/store/postgres/src/block_range.rs +++ b/store/postgres/src/block_range.rs @@ -50,7 +50,7 @@ lazy_static! { /// The range of blocks for which an entity is valid. We need this struct /// to bind ranges into Diesel queries. -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Copy)] pub struct BlockRange(Bound, Bound); pub(crate) fn first_block_in_range( @@ -147,6 +147,16 @@ pub enum BlockRangeColumn<'a> { table_prefix: &'a str, block: BlockNumber, }, + MutableRange { + table: &'a Table, + table_prefix: &'a str, + block_range: BlockRange, // TODO: check if this is a proper type here (maybe Range?) + }, + ImmutableRange { + table: &'a Table, + table_prefix: &'a str, + block_range: BlockRange, + }, } impl<'a> BlockRangeColumn<'a> { @@ -166,10 +176,35 @@ impl<'a> BlockRangeColumn<'a> { } } + // TODO: refactor new and new2 into one. use enum of both BlockNumber and range + pub fn new2( + table: &'a Table, + table_prefix: &'a str, + block_range: std::ops::Range, + ) -> Self { + let st: Bound = Bound::Included(block_range.start.try_into().unwrap()); + let en: Bound = Bound::Excluded(block_range.end.try_into().unwrap()); + let block_range: BlockRange = BlockRange(st, en); + if table.immutable { + Self::ImmutableRange { + table, + table_prefix, + block_range, + } + } else { + Self::MutableRange { + table, + table_prefix, + block_range, + } + } + } + pub fn block(&self) -> BlockNumber { match self { BlockRangeColumn::Mutable { block, .. } => *block, BlockRangeColumn::Immutable { block, .. } => *block, + _ => todo!(), } } } @@ -224,6 +259,40 @@ impl<'a> BlockRangeColumn<'a> { out.push_bind_param::(block) } } + BlockRangeColumn::MutableRange { + table: _, + table_prefix: _, + block_range: BlockRange(start, finish), + } => { + out.push_sql("block_range && int4range("); + match start { + Bound::Included(block) => out.push_bind_param::(block)?, + Bound::Excluded(block) => { + out.push_bind_param::(block)?; + out.push_sql("+1"); + } + Bound::Unbounded => todo!(), + }; + out.push_sql(","); + match finish { + Bound::Included(block) => { + out.push_bind_param::(block)?; + out.push_sql("+1"); + } + Bound::Excluded(block) => out.push_bind_param::(block)?, + Bound::Unbounded => todo!(), + }; + out.push_sql(")"); + Ok(()) + } + BlockRangeColumn::ImmutableRange { + table: _, + table_prefix: _, + block_range: _, + } => { + println!("ImmutableRange conatins()"); + todo!() + } } } @@ -231,6 +300,7 @@ impl<'a> BlockRangeColumn<'a> { match self { BlockRangeColumn::Mutable { .. } => BLOCK_RANGE_COLUMN, BlockRangeColumn::Immutable { .. } => BLOCK_COLUMN, + _ => todo!(), } } @@ -245,6 +315,7 @@ impl<'a> BlockRangeColumn<'a> { out.push_sql(table_prefix); out.push_sql(BLOCK_COLUMN); } + _ => todo!(), } } @@ -254,6 +325,7 @@ impl<'a> BlockRangeColumn<'a> { match self { BlockRangeColumn::Mutable { .. } => out.push_sql(BLOCK_RANGE_CURRENT), BlockRangeColumn::Immutable { .. } => out.push_sql("true"), + _ => todo!(), } } @@ -277,6 +349,7 @@ impl<'a> BlockRangeColumn<'a> { BlockRangeColumn::Immutable { .. } => { unreachable!("immutable entities can not be updated or deleted") } + _ => todo!(), } } @@ -285,6 +358,7 @@ impl<'a> BlockRangeColumn<'a> { match self { BlockRangeColumn::Mutable { .. } => out.push_sql(BLOCK_RANGE_COLUMN), BlockRangeColumn::Immutable { .. } => out.push_sql(BLOCK_COLUMN), + _ => todo!(), } } @@ -303,6 +377,7 @@ impl<'a> BlockRangeColumn<'a> { out.push_sql(" >= "); out.push_bind_param::(block) } + _ => todo!(), } } } diff --git a/store/postgres/src/deployment_store.rs b/store/postgres/src/deployment_store.rs index d8b04faac0b..eb73c5614b3 100644 --- a/store/postgres/src/deployment_store.rs +++ b/store/postgres/src/deployment_store.rs @@ -29,8 +29,8 @@ use lru_time_cache::LruCache; use rand::{seq::SliceRandom, thread_rng}; use std::collections::{BTreeMap, HashMap}; use std::convert::Into; -use std::ops::Deref; use std::ops::{Bound, DerefMut}; +use std::ops::{Deref, Range}; use std::str::FromStr; use std::sync::{atomic::AtomicUsize, Arc, Mutex}; use std::time::{Duration, Instant}; @@ -1056,6 +1056,17 @@ impl DeploymentStore { layout.find_many(&mut conn, ids_for_type, block) } + pub(crate) fn get_range( + &self, + site: Arc, + key: &EntityKey, + block_range: Range, + ) -> Result, StoreError> { + let mut conn = self.get_conn()?; + let layout = self.layout(&mut conn, site)?; + layout.find_range(&mut conn, key, block_range) + } + pub(crate) fn get_derived( &self, site: Arc, diff --git a/store/postgres/src/relational.rs b/store/postgres/src/relational.rs index 8ceb8d9c714..1dba3e6e502 100644 --- a/store/postgres/src/relational.rs +++ b/store/postgres/src/relational.rs @@ -46,6 +46,7 @@ use std::borrow::Borrow; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::convert::{From, TryFrom}; use std::fmt::{self, Write}; +use std::ops::Range; use std::str::FromStr; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; @@ -58,7 +59,7 @@ use crate::{ primary::{Namespace, Site}, relational_queries::{ ClampRangeQuery, EntityData, EntityDeletion, FilterCollection, FilterQuery, FindManyQuery, - FindQuery, InsertQuery, RevertClampQuery, RevertRemoveQuery, + FindQuery, FindRangeQuery, InsertQuery, RevertClampQuery, RevertRemoveQuery, }, }; use graph::components::store::DerivedEntityQuery; @@ -514,6 +515,27 @@ impl Layout { Ok(entities) } + pub fn find_range( + &self, + conn: &mut PgConnection, + key: &EntityKey, + block_range: Range, + ) -> Result, StoreError> { + let table = self.table_for_entity(&key.entity_type)?; + let mut entities: BTreeMap = BTreeMap::new(); + if let Some(vec) = FindRangeQuery::new(table.as_ref(), key, block_range) + .get_results::(conn) + .optional()? + { + for e in vec { + let block = e.clone().deserialize_block_number::()?; + let en = e.deserialize_with_layout::(self, None)?; + entities.insert(block, en); + } + } + Ok(entities) + } + pub fn find_derived( &self, conn: &mut PgConnection, diff --git a/store/postgres/src/relational_queries.rs b/store/postgres/src/relational_queries.rs index 4626ce0479e..41d5937ecfe 100644 --- a/store/postgres/src/relational_queries.rs +++ b/store/postgres/src/relational_queries.rs @@ -17,6 +17,7 @@ use graph::data::store::{Id, IdType, NULL}; use graph::data::store::{IdList, IdRef, QueryObject}; use graph::data::value::{Object, Word}; use graph::data_source::CausalityRegion; +use graph::prelude::regex::Regex; use graph::prelude::{ anyhow, r, serde_json, BlockNumber, ChildMultiplicity, Entity, EntityCollection, EntityFilter, EntityLink, EntityOrder, EntityOrderByChild, EntityOrderByChildInfo, EntityRange, EntityWindow, @@ -30,6 +31,7 @@ use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::convert::TryFrom; use std::fmt::{self, Display}; use std::iter::FromIterator; +use std::ops::Range; use std::str::FromStr; use std::string::ToString; @@ -466,7 +468,7 @@ pub fn parse_id(id_type: IdType, json: serde_json::Value) -> Result(self) -> Result { + use serde_json::Value as j; + match self.data { + j::Object(map) => { + let mut entries = map.into_iter().filter_map(move |(key, json)| { + if key == "block_range" { + let r = json.as_str().unwrap(); + let rx = Regex::new("\\[(?P[0-9]+),[0-9]+\\)").unwrap(); + let cap = rx.captures(r).unwrap(); + let start = cap + .name("start") + .map(|mtch| mtch.as_str().to_string()) + .unwrap(); + let n = start.parse::().unwrap(); + Some(n) + } else { + None + } + }); + let en = entries.next().unwrap(); + assert!(entries.next().is_none()); // there should be just one block_range field + Ok(en) + } + _ => unreachable!( + "we use `to_json` in our queries, and will therefore always get an object back" + ), + } + } + /// Map the `EntityData` using the schema information in `Layout` pub fn deserialize_with_layout( self, @@ -1971,6 +2002,65 @@ impl<'a> Query for FindQuery<'a> { impl<'a, Conn> RunQueryDsl for FindQuery<'a> {} +#[derive(Debug, Clone)] +pub struct FindRangeQuery<'a> { + table: &'a Table, + key: &'a EntityKey, + br_column: BlockRangeColumn<'a>, +} + +impl<'a> FindRangeQuery<'a> { + pub fn new(table: &'a Table, key: &'a EntityKey, block_range: Range) -> Self { + let br_column = BlockRangeColumn::new2(table, "e.", block_range); + Self { + table, + key, + br_column, + } + } +} + +impl<'a> QueryFragment for FindRangeQuery<'a> { + fn walk_ast<'b>(&'b self, mut out: AstPass<'_, 'b, Pg>) -> QueryResult<()> { + out.unsafe_to_cache_prepared(); + + // Generate + // select '..' as entity, to_jsonb(e.*) as data + // from schema.table e where id = $1 + + // out.push_sql("\nunion all\n"); + + out.push_sql("select "); + // out.push_sql("e.* "); + out.push_bind_param::(self.table.object.as_str())?; + out.push_sql(" as entity, to_jsonb(e.*) as data\n"); + // out.push_sql(" select to_jsonb(e.*) as data\n"); + out.push_sql(" from "); + out.push_sql(self.table.qualified_name.as_str()); + out.push_sql(" e\n where "); + self.table.primary_key().eq(&self.key.entity_id, &mut out)?; + out.push_sql(" and "); + if self.table.has_causality_region { + out.push_sql("causality_region = "); + out.push_bind_param::(&self.key.causality_region)?; + out.push_sql(" and "); + } + self.br_column.contains(&mut out, true) + } +} + +impl<'a> QueryId for FindRangeQuery<'a> { + type QueryId = (); + + const HAS_STATIC_QUERY_ID: bool = false; +} + +impl<'a> Query for FindRangeQuery<'a> { + type SqlType = Untyped; +} + +impl<'a, Conn> RunQueryDsl for FindRangeQuery<'a> {} + /// Builds a query over a given set of [`Table`]s in an attempt to find updated /// and/or newly inserted entities at a given block number; i.e. such that the /// block range's lower bound is equal to said block number. diff --git a/store/postgres/src/writable.rs b/store/postgres/src/writable.rs index ee7a5e4754f..dbda05a78ff 100644 --- a/store/postgres/src/writable.rs +++ b/store/postgres/src/writable.rs @@ -1,5 +1,5 @@ use std::collections::BTreeSet; -use std::ops::Deref; +use std::ops::{Deref, Range}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Mutex, RwLock, TryLockError as RwLockError}; use std::time::Instant; @@ -352,6 +352,17 @@ impl SyncStore { }) } + fn get_range( + &self, + key: &EntityKey, + block_range: Range, + ) -> Result, StoreError> { + retry::forever(&self.logger, "get_range", || { + self.writable + .get_range(self.site.cheap_clone(), key, block_range.clone()) + }) + } + fn get_derived( &self, key: &DerivedEntityQuery, @@ -1217,6 +1228,15 @@ impl Queue { Ok(map) } + fn get_range( + &self, + key: &EntityKey, + block_range: Range, + ) -> Result, StoreError> { + // TODO: implemet read from the queue + self.store.get_range(key, block_range) + } + fn get_derived( &self, derived_query: &DerivedEntityQuery, @@ -1429,6 +1449,17 @@ impl Writer { } } + fn get_range( + &self, + key: &EntityKey, + block_range: Range, + ) -> Result, StoreError> { + match self { + Writer::Sync(store) => store.get_range(key, block_range), + Writer::Async { queue, .. } => queue.get_range(key, block_range), + } + } + fn get_derived( &self, key: &DerivedEntityQuery, @@ -1557,6 +1588,14 @@ impl ReadStore for WritableStore { self.writer.get_many(keys) } + fn get_range( + &self, + key: &EntityKey, + block_range: Range, + ) -> Result, StoreError> { + self.writer.get_range(key, block_range) + } + fn get_derived( &self, key: &DerivedEntityQuery, diff --git a/store/test-store/tests/graph/entity_cache.rs b/store/test-store/tests/graph/entity_cache.rs index d7ebb30785c..9efb5abcb1b 100644 --- a/store/test-store/tests/graph/entity_cache.rs +++ b/store/test-store/tests/graph/entity_cache.rs @@ -20,6 +20,7 @@ use lazy_static::lazy_static; use slog::Logger; use std::collections::{BTreeMap, BTreeSet}; use std::marker::PhantomData; +use std::ops::Range; use std::sync::Arc; use web3::types::H256; @@ -66,6 +67,14 @@ impl ReadStore for MockStore { Ok(self.get_many_res.clone()) } + fn get_range( + &self, + _key: &EntityKey, + _block_range: Range, + ) -> Result, StoreError> { + todo!() + } + fn get_derived( &self, _key: &DerivedEntityQuery, diff --git a/store/test-store/tests/postgres/writable.rs b/store/test-store/tests/postgres/writable.rs index 4a986e6f3fa..d2f4385fb7c 100644 --- a/store/test-store/tests/postgres/writable.rs +++ b/store/test-store/tests/postgres/writable.rs @@ -6,6 +6,7 @@ use graph::schema::{EntityKey, EntityType, InputSchema}; use lazy_static::lazy_static; use std::collections::BTreeSet; use std::marker::PhantomData; +use std::ops::Range; use test_store::*; use graph::components::store::{DeploymentLocator, DerivedEntityQuery, WritableStore}; @@ -111,7 +112,12 @@ fn count_key(id: &str) -> EntityKey { COUNTER_TYPE.parse_key(id).unwrap() } -async fn insert_count(store: &Arc, deployment: &DeploymentLocator, count: u8) { +async fn insert_count( + store: &Arc, + deployment: &DeploymentLocator, + block: u8, + count: u8, +) { let data = entity! { TEST_SUBGRAPH_SCHEMA => id: "1", count: count as i32 @@ -120,7 +126,7 @@ async fn insert_count(store: &Arc, deployment: &DeploymentL key: count_key(&data.get("id").unwrap().to_string()), data, }; - transact_entity_operations(store, deployment, block_pointer(count), vec![entity_op]) + transact_entity_operations(store, deployment, block_pointer(block), vec![entity_op]) .await .unwrap(); } @@ -150,13 +156,13 @@ where } for count in 1..4 { - insert_count(&subgraph_store, &deployment, count).await; + insert_count(&subgraph_store, &deployment, count, count).await; } // Test reading back with pending writes to the same entity pause_writer(&deployment).await; for count in 4..7 { - insert_count(&subgraph_store, &deployment, count).await; + insert_count(&subgraph_store, &deployment, count, count).await; } assert_eq!(6, read_count()); @@ -165,7 +171,7 @@ where // Test reading back with pending writes and a pending revert for count in 7..10 { - insert_count(&subgraph_store, &deployment, count).await; + insert_count(&subgraph_store, &deployment, count, count).await; } writable .revert_block_operations(block_pointer(2), FirehoseCursor::None) @@ -286,3 +292,21 @@ fn restart() { writable.flush().await.unwrap(); }) } + +#[test] +fn entities_read_range() { + run_test(|store, writable, deployment| async move { + let subgraph_store = store.subgraph_store(); + let read_count = || count_get(writable.as_ref()); + writable.deployment_synced().unwrap(); + + for count in 1..7 { + insert_count(&subgraph_store, &deployment, count, 2 * count).await; + } + writable.flush().await.unwrap(); + assert_eq!(2 * (7 - 1), read_count()); + let br: Range = 2..5; + let e = writable.get_range(&count_key("1"), br).unwrap(); + assert_eq!(e.len(), 5 - 2) + }) +} From dd7c1cabf4c2ff574fc3e6dc7028a83cf907eae6 Mon Sep 17 00:00:00 2001 From: Zoran Cvetkov Date: Thu, 18 Jul 2024 16:56:13 +0300 Subject: [PATCH 21/29] remove the key as a parameter --- graph/src/components/store/mod.rs | 2 +- graph/src/components/store/traits.rs | 6 ++--- store/postgres/src/deployment_store.rs | 4 +-- store/postgres/src/relational.rs | 6 ++--- store/postgres/src/relational_queries.rs | 26 ++++++-------------- store/postgres/src/writable.rs | 18 +++++++------- store/test-store/tests/graph/entity_cache.rs | 2 +- store/test-store/tests/postgres/writable.rs | 3 ++- 8 files changed, 28 insertions(+), 39 deletions(-) diff --git a/graph/src/components/store/mod.rs b/graph/src/components/store/mod.rs index edf3f575512..f978e92edb5 100644 --- a/graph/src/components/store/mod.rs +++ b/graph/src/components/store/mod.rs @@ -1041,7 +1041,7 @@ impl ReadStore for EmptyStore { fn get_range( &self, - _key: &EntityKey, + _entity_type: &EntityType, _block_range: Range, ) -> Result, StoreError> { Ok(BTreeMap::new()) diff --git a/graph/src/components/store/traits.rs b/graph/src/components/store/traits.rs index 6cc8ff23ec1..5557ae3fdd0 100644 --- a/graph/src/components/store/traits.rs +++ b/graph/src/components/store/traits.rs @@ -231,7 +231,7 @@ pub trait ReadStore: Send + Sync + 'static { /// Looks up entities using the given store key for a range of blocks. fn get_range( &self, - key: &EntityKey, + entity_type: &EntityType, block_range: Range, ) -> Result, StoreError>; @@ -259,10 +259,10 @@ impl ReadStore for Arc { fn get_range( &self, - key: &EntityKey, + entity_type: &EntityType, block_range: Range, ) -> Result, StoreError> { - (**self).get_range(key, block_range) + (**self).get_range(entity_type, block_range) } fn get_derived( diff --git a/store/postgres/src/deployment_store.rs b/store/postgres/src/deployment_store.rs index eb73c5614b3..3b4327afb97 100644 --- a/store/postgres/src/deployment_store.rs +++ b/store/postgres/src/deployment_store.rs @@ -1059,12 +1059,12 @@ impl DeploymentStore { pub(crate) fn get_range( &self, site: Arc, - key: &EntityKey, + entity_type: &EntityType, block_range: Range, ) -> Result, StoreError> { let mut conn = self.get_conn()?; let layout = self.layout(&mut conn, site)?; - layout.find_range(&mut conn, key, block_range) + layout.find_range(&mut conn, entity_type, block_range) } pub(crate) fn get_derived( diff --git a/store/postgres/src/relational.rs b/store/postgres/src/relational.rs index 1dba3e6e502..379e8f020e1 100644 --- a/store/postgres/src/relational.rs +++ b/store/postgres/src/relational.rs @@ -518,12 +518,12 @@ impl Layout { pub fn find_range( &self, conn: &mut PgConnection, - key: &EntityKey, + entity_type: &EntityType, block_range: Range, ) -> Result, StoreError> { - let table = self.table_for_entity(&key.entity_type)?; + let table = self.table_for_entity(entity_type)?; let mut entities: BTreeMap = BTreeMap::new(); - if let Some(vec) = FindRangeQuery::new(table.as_ref(), key, block_range) + if let Some(vec) = FindRangeQuery::new(table.as_ref(), block_range) .get_results::(conn) .optional()? { diff --git a/store/postgres/src/relational_queries.rs b/store/postgres/src/relational_queries.rs index 41d5937ecfe..b1609448d07 100644 --- a/store/postgres/src/relational_queries.rs +++ b/store/postgres/src/relational_queries.rs @@ -2005,18 +2005,13 @@ impl<'a, Conn> RunQueryDsl for FindQuery<'a> {} #[derive(Debug, Clone)] pub struct FindRangeQuery<'a> { table: &'a Table, - key: &'a EntityKey, br_column: BlockRangeColumn<'a>, } impl<'a> FindRangeQuery<'a> { - pub fn new(table: &'a Table, key: &'a EntityKey, block_range: Range) -> Self { + pub fn new(table: &'a Table, block_range: Range) -> Self { let br_column = BlockRangeColumn::new2(table, "e.", block_range); - Self { - table, - key, - br_column, - } + Self { table, br_column } } } @@ -2027,24 +2022,17 @@ impl<'a> QueryFragment for FindRangeQuery<'a> { // Generate // select '..' as entity, to_jsonb(e.*) as data // from schema.table e where id = $1 - - // out.push_sql("\nunion all\n"); - out.push_sql("select "); - // out.push_sql("e.* "); out.push_bind_param::(self.table.object.as_str())?; out.push_sql(" as entity, to_jsonb(e.*) as data\n"); - // out.push_sql(" select to_jsonb(e.*) as data\n"); out.push_sql(" from "); out.push_sql(self.table.qualified_name.as_str()); out.push_sql(" e\n where "); - self.table.primary_key().eq(&self.key.entity_id, &mut out)?; - out.push_sql(" and "); - if self.table.has_causality_region { - out.push_sql("causality_region = "); - out.push_bind_param::(&self.key.causality_region)?; - out.push_sql(" and "); - } + // if self.table.has_causality_region { + // out.push_sql("causality_region = "); + // out.push_bind_param::(&self.key.causality_region)?; + // out.push_sql(" and "); + // } self.br_column.contains(&mut out, true) } } diff --git a/store/postgres/src/writable.rs b/store/postgres/src/writable.rs index dbda05a78ff..8a99c7761e8 100644 --- a/store/postgres/src/writable.rs +++ b/store/postgres/src/writable.rs @@ -354,12 +354,12 @@ impl SyncStore { fn get_range( &self, - key: &EntityKey, + entity_type: &EntityType, block_range: Range, ) -> Result, StoreError> { retry::forever(&self.logger, "get_range", || { self.writable - .get_range(self.site.cheap_clone(), key, block_range.clone()) + .get_range(self.site.cheap_clone(), entity_type, block_range.clone()) }) } @@ -1230,11 +1230,11 @@ impl Queue { fn get_range( &self, - key: &EntityKey, + entity_type: &EntityType, block_range: Range, ) -> Result, StoreError> { // TODO: implemet read from the queue - self.store.get_range(key, block_range) + self.store.get_range(entity_type, block_range) } fn get_derived( @@ -1451,12 +1451,12 @@ impl Writer { fn get_range( &self, - key: &EntityKey, + entity_type: &EntityType, block_range: Range, ) -> Result, StoreError> { match self { - Writer::Sync(store) => store.get_range(key, block_range), - Writer::Async { queue, .. } => queue.get_range(key, block_range), + Writer::Sync(store) => store.get_range(entity_type, block_range), + Writer::Async { queue, .. } => queue.get_range(entity_type, block_range), } } @@ -1590,10 +1590,10 @@ impl ReadStore for WritableStore { fn get_range( &self, - key: &EntityKey, + entity_type: &EntityType, block_range: Range, ) -> Result, StoreError> { - self.writer.get_range(key, block_range) + self.writer.get_range(entity_type, block_range) } fn get_derived( diff --git a/store/test-store/tests/graph/entity_cache.rs b/store/test-store/tests/graph/entity_cache.rs index 9efb5abcb1b..d53b5464614 100644 --- a/store/test-store/tests/graph/entity_cache.rs +++ b/store/test-store/tests/graph/entity_cache.rs @@ -69,7 +69,7 @@ impl ReadStore for MockStore { fn get_range( &self, - _key: &EntityKey, + _entity_type: &EntityType, _block_range: Range, ) -> Result, StoreError> { todo!() diff --git a/store/test-store/tests/postgres/writable.rs b/store/test-store/tests/postgres/writable.rs index d2f4385fb7c..f91bff0d126 100644 --- a/store/test-store/tests/postgres/writable.rs +++ b/store/test-store/tests/postgres/writable.rs @@ -306,7 +306,8 @@ fn entities_read_range() { writable.flush().await.unwrap(); assert_eq!(2 * (7 - 1), read_count()); let br: Range = 2..5; - let e = writable.get_range(&count_key("1"), br).unwrap(); + let et = &COUNTER_TYPE; + let e = writable.get_range(&et, br).unwrap(); assert_eq!(e.len(), 5 - 2) }) } From f54e705da72ff19e732b083a423467fa7f1eefe1 Mon Sep 17 00:00:00 2001 From: Zoran Cvetkov Date: Wed, 24 Jul 2024 17:14:44 +0300 Subject: [PATCH 22/29] fix regex --- store/postgres/src/relational_queries.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/store/postgres/src/relational_queries.rs b/store/postgres/src/relational_queries.rs index b1609448d07..b19e2085084 100644 --- a/store/postgres/src/relational_queries.rs +++ b/store/postgres/src/relational_queries.rs @@ -488,7 +488,7 @@ impl EntityData { let mut entries = map.into_iter().filter_map(move |(key, json)| { if key == "block_range" { let r = json.as_str().unwrap(); - let rx = Regex::new("\\[(?P[0-9]+),[0-9]+\\)").unwrap(); + let rx = Regex::new("\\[(?P[0-9]+),([0-9]+)?\\)").unwrap(); let cap = rx.captures(r).unwrap(); let start = cap .name("start") From fbe9612bbc0f2f6830388461623d374c6b344b6e Mon Sep 17 00:00:00 2001 From: Zoran Cvetkov Date: Tue, 30 Jul 2024 16:39:27 +0300 Subject: [PATCH 23/29] proper return type --- graph/src/components/store/mod.rs | 2 +- graph/src/components/store/traits.rs | 4 ++-- store/postgres/src/deployment_store.rs | 2 +- store/postgres/src/relational.rs | 11 ++++++++--- store/postgres/src/writable.rs | 8 ++++---- store/test-store/tests/graph/entity_cache.rs | 2 +- 6 files changed, 17 insertions(+), 12 deletions(-) diff --git a/graph/src/components/store/mod.rs b/graph/src/components/store/mod.rs index f978e92edb5..e0992f9d62d 100644 --- a/graph/src/components/store/mod.rs +++ b/graph/src/components/store/mod.rs @@ -1043,7 +1043,7 @@ impl ReadStore for EmptyStore { &self, _entity_type: &EntityType, _block_range: Range, - ) -> Result, StoreError> { + ) -> Result>, StoreError> { Ok(BTreeMap::new()) } diff --git a/graph/src/components/store/traits.rs b/graph/src/components/store/traits.rs index 5557ae3fdd0..a916a5e7241 100644 --- a/graph/src/components/store/traits.rs +++ b/graph/src/components/store/traits.rs @@ -233,7 +233,7 @@ pub trait ReadStore: Send + Sync + 'static { &self, entity_type: &EntityType, block_range: Range, - ) -> Result, StoreError>; + ) -> Result>, StoreError>; /// Reverse lookup fn get_derived( @@ -261,7 +261,7 @@ impl ReadStore for Arc { &self, entity_type: &EntityType, block_range: Range, - ) -> Result, StoreError> { + ) -> Result>, StoreError> { (**self).get_range(entity_type, block_range) } diff --git a/store/postgres/src/deployment_store.rs b/store/postgres/src/deployment_store.rs index 3b4327afb97..1dac6c46279 100644 --- a/store/postgres/src/deployment_store.rs +++ b/store/postgres/src/deployment_store.rs @@ -1061,7 +1061,7 @@ impl DeploymentStore { site: Arc, entity_type: &EntityType, block_range: Range, - ) -> Result, StoreError> { + ) -> Result>, StoreError> { let mut conn = self.get_conn()?; let layout = self.layout(&mut conn, site)?; layout.find_range(&mut conn, entity_type, block_range) diff --git a/store/postgres/src/relational.rs b/store/postgres/src/relational.rs index 379e8f020e1..5f10c4bc8e9 100644 --- a/store/postgres/src/relational.rs +++ b/store/postgres/src/relational.rs @@ -520,9 +520,9 @@ impl Layout { conn: &mut PgConnection, entity_type: &EntityType, block_range: Range, - ) -> Result, StoreError> { + ) -> Result>, StoreError> { let table = self.table_for_entity(entity_type)?; - let mut entities: BTreeMap = BTreeMap::new(); + let mut entities: BTreeMap> = BTreeMap::new(); if let Some(vec) = FindRangeQuery::new(table.as_ref(), block_range) .get_results::(conn) .optional()? @@ -530,7 +530,12 @@ impl Layout { for e in vec { let block = e.clone().deserialize_block_number::()?; let en = e.deserialize_with_layout::(self, None)?; - entities.insert(block, en); + match entities.get_mut(&block) { + Some(vec) => vec.push(en), + None => { + let _ = entities.insert(block, vec![en]); + } + }; } } Ok(entities) diff --git a/store/postgres/src/writable.rs b/store/postgres/src/writable.rs index 8a99c7761e8..a1b1241a15a 100644 --- a/store/postgres/src/writable.rs +++ b/store/postgres/src/writable.rs @@ -356,7 +356,7 @@ impl SyncStore { &self, entity_type: &EntityType, block_range: Range, - ) -> Result, StoreError> { + ) -> Result>, StoreError> { retry::forever(&self.logger, "get_range", || { self.writable .get_range(self.site.cheap_clone(), entity_type, block_range.clone()) @@ -1232,7 +1232,7 @@ impl Queue { &self, entity_type: &EntityType, block_range: Range, - ) -> Result, StoreError> { + ) -> Result>, StoreError> { // TODO: implemet read from the queue self.store.get_range(entity_type, block_range) } @@ -1453,7 +1453,7 @@ impl Writer { &self, entity_type: &EntityType, block_range: Range, - ) -> Result, StoreError> { + ) -> Result>, StoreError> { match self { Writer::Sync(store) => store.get_range(entity_type, block_range), Writer::Async { queue, .. } => queue.get_range(entity_type, block_range), @@ -1592,7 +1592,7 @@ impl ReadStore for WritableStore { &self, entity_type: &EntityType, block_range: Range, - ) -> Result, StoreError> { + ) -> Result>, StoreError> { self.writer.get_range(entity_type, block_range) } diff --git a/store/test-store/tests/graph/entity_cache.rs b/store/test-store/tests/graph/entity_cache.rs index d53b5464614..b1f0d9a0f23 100644 --- a/store/test-store/tests/graph/entity_cache.rs +++ b/store/test-store/tests/graph/entity_cache.rs @@ -71,7 +71,7 @@ impl ReadStore for MockStore { &self, _entity_type: &EntityType, _block_range: Range, - ) -> Result, StoreError> { + ) -> Result>, StoreError> { todo!() } From 8b0f3f3ee4ddf60c3eb1fa91041689e0b541ae28 Mon Sep 17 00:00:00 2001 From: Zoran Cvetkov Date: Fri, 2 Aug 2024 20:36:35 +0300 Subject: [PATCH 24/29] support immutable entities --- store/postgres/src/block_range.rs | 23 ++++++++++++++++++++--- store/postgres/src/relational_queries.rs | 3 +++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/store/postgres/src/block_range.rs b/store/postgres/src/block_range.rs index b6d11811917..936afdd7025 100644 --- a/store/postgres/src/block_range.rs +++ b/store/postgres/src/block_range.rs @@ -288,10 +288,27 @@ impl<'a> BlockRangeColumn<'a> { BlockRangeColumn::ImmutableRange { table: _, table_prefix: _, - block_range: _, + block_range: BlockRange(start, finish), } => { - println!("ImmutableRange conatins()"); - todo!() + out.push_sql("block$ >= "); + match start { + Bound::Included(block) => out.push_bind_param::(block)?, + Bound::Excluded(block) => { + out.push_bind_param::(block)?; + out.push_sql("+1"); + } + Bound::Unbounded => todo!(), + }; + out.push_sql(" AND block$ <= "); + match finish { + Bound::Included(block) => { + out.push_bind_param::(block)?; + out.push_sql("+1"); + } + Bound::Excluded(block) => out.push_bind_param::(block)?, + Bound::Unbounded => todo!(), + }; + Ok(()) } } } diff --git a/store/postgres/src/relational_queries.rs b/store/postgres/src/relational_queries.rs index b19e2085084..d4f205a4bc2 100644 --- a/store/postgres/src/relational_queries.rs +++ b/store/postgres/src/relational_queries.rs @@ -496,6 +496,9 @@ impl EntityData { .unwrap(); let n = start.parse::().unwrap(); Some(n) + } else if key == "block$" { + let block = json.as_i64().unwrap() as i32; + Some(block) } else { None } From 0119e45ba2d074d85935106c8a596b2c6f86895e Mon Sep 17 00:00:00 2001 From: Zoran Cvetkov Date: Mon, 5 Aug 2024 20:45:40 +0300 Subject: [PATCH 25/29] add test for immutable entities --- store/test-store/tests/postgres/writable.rs | 98 +++++++++++++++++---- 1 file changed, 79 insertions(+), 19 deletions(-) diff --git a/store/test-store/tests/postgres/writable.rs b/store/test-store/tests/postgres/writable.rs index f91bff0d126..26545ba2f00 100644 --- a/store/test-store/tests/postgres/writable.rs +++ b/store/test-store/tests/postgres/writable.rs @@ -22,9 +22,14 @@ const SCHEMA_GQL: &str = " id: ID!, count: Int!, } + type Counter2 @entity(immutable: true) { + id: ID!, + count: Int!, + } "; const COUNTER: &str = "Counter"; +const COUNTER2: &str = "Counter2"; lazy_static! { static ref TEST_SUBGRAPH_ID_STRING: String = String::from("writableSubgraph"); @@ -34,6 +39,7 @@ lazy_static! { InputSchema::parse_latest(SCHEMA_GQL, TEST_SUBGRAPH_ID.clone()) .expect("Failed to parse user schema"); static ref COUNTER_TYPE: EntityType = TEST_SUBGRAPH_SCHEMA.entity_type(COUNTER).unwrap(); + static ref COUNTER2_TYPE: EntityType = TEST_SUBGRAPH_SCHEMA.entity_type(COUNTER2).unwrap(); } /// Inserts test data into the store. @@ -117,13 +123,16 @@ async fn insert_count( deployment: &DeploymentLocator, block: u8, count: u8, + counter_type: &EntityType, + id: &str, ) { + let count_key_local = |id: &str| counter_type.parse_key(id).unwrap(); let data = entity! { TEST_SUBGRAPH_SCHEMA => - id: "1", - count: count as i32 + id: id, + count :count as i32, }; let entity_op = EntityOperation::Set { - key: count_key(&data.get("id").unwrap().to_string()), + key: count_key_local(&data.get("id").unwrap().to_string()), data, }; transact_entity_operations(store, deployment, block_pointer(block), vec![entity_op]) @@ -131,6 +140,32 @@ async fn insert_count( .unwrap(); } +async fn insert_count_mutable( + store: &Arc, + deployment: &DeploymentLocator, + block: u8, + count: u8, +) { + insert_count(store, deployment, block, count, &COUNTER_TYPE, "1").await; +} + +async fn insert_count_immutable( + store: &Arc, + deployment: &DeploymentLocator, + block: u8, + count: u8, +) { + insert_count( + store, + deployment, + block, + count, + &COUNTER2_TYPE, + &(block / 2).to_string(), + ) + .await; +} + async fn pause_writer(deployment: &DeploymentLocator) { flush(deployment).await.unwrap(); writable::allow_steps(deployment, 0).await; @@ -156,13 +191,13 @@ where } for count in 1..4 { - insert_count(&subgraph_store, &deployment, count, count).await; + insert_count_mutable(&subgraph_store, &deployment, count, count).await; } // Test reading back with pending writes to the same entity pause_writer(&deployment).await; for count in 4..7 { - insert_count(&subgraph_store, &deployment, count, count).await; + insert_count_mutable(&subgraph_store, &deployment, count, count).await; } assert_eq!(6, read_count()); @@ -171,7 +206,7 @@ where // Test reading back with pending writes and a pending revert for count in 7..10 { - insert_count(&subgraph_store, &deployment, count, count).await; + insert_count_mutable(&subgraph_store, &deployment, count, count).await; } writable .revert_block_operations(block_pointer(2), FirehoseCursor::None) @@ -293,21 +328,46 @@ fn restart() { }) } +async fn read_range( + store: Arc, + writable: Arc, + deployment: DeploymentLocator, + mutable: bool, +) -> usize { + let subgraph_store = store.subgraph_store(); + writable.deployment_synced().unwrap(); + + for count in 1..=7 { + if mutable { + insert_count_mutable(&subgraph_store, &deployment, 2 * count, 4 * count).await + } else { + insert_count_immutable(&subgraph_store, &deployment, 2 * count, 4 * count).await + } + } + writable.flush().await.unwrap(); + + let br: Range = 4..8; + let et: &EntityType = if mutable { + &COUNTER_TYPE + } else { + &COUNTER2_TYPE + }; + let e = writable.get_range(et, br).unwrap(); + e.len() +} + #[test] -fn entities_read_range() { +fn read_range_mutable() { run_test(|store, writable, deployment| async move { - let subgraph_store = store.subgraph_store(); - let read_count = || count_get(writable.as_ref()); - writable.deployment_synced().unwrap(); + let num_entities = read_range(store, writable, deployment, true).await; + assert_eq!(num_entities, 2) + }) +} - for count in 1..7 { - insert_count(&subgraph_store, &deployment, count, 2 * count).await; - } - writable.flush().await.unwrap(); - assert_eq!(2 * (7 - 1), read_count()); - let br: Range = 2..5; - let et = &COUNTER_TYPE; - let e = writable.get_range(&et, br).unwrap(); - assert_eq!(e.len(), 5 - 2) +#[test] +fn read_range_immutable() { + run_test(|store, writable, deployment| async move { + let num_entities = read_range(store, writable, deployment, false).await; + assert_eq!(num_entities, 3) // TODO: fix it - it should be 2 as the range is open }) } From 2bf501818900d00702f7b4d5139c6eb94401a64f Mon Sep 17 00:00:00 2001 From: incrypto32 Date: Wed, 31 Jul 2024 09:20:03 +0530 Subject: [PATCH 26/29] store: fix block range query bug when getting entity triggers --- store/postgres/src/block_range.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/store/postgres/src/block_range.rs b/store/postgres/src/block_range.rs index 936afdd7025..2da371684eb 100644 --- a/store/postgres/src/block_range.rs +++ b/store/postgres/src/block_range.rs @@ -264,7 +264,7 @@ impl<'a> BlockRangeColumn<'a> { table_prefix: _, block_range: BlockRange(start, finish), } => { - out.push_sql("block_range && int4range("); + out.push_sql("lower(block_range) >= "); match start { Bound::Included(block) => out.push_bind_param::(block)?, Bound::Excluded(block) => { @@ -273,7 +273,7 @@ impl<'a> BlockRangeColumn<'a> { } Bound::Unbounded => todo!(), }; - out.push_sql(","); + out.push_sql(" AND lower(block_range) <= "); match finish { Bound::Included(block) => { out.push_bind_param::(block)?; @@ -282,7 +282,6 @@ impl<'a> BlockRangeColumn<'a> { Bound::Excluded(block) => out.push_bind_param::(block)?, Bound::Unbounded => todo!(), }; - out.push_sql(")"); Ok(()) } BlockRangeColumn::ImmutableRange { From 52a5d01abf0d5a8c3d4c1b114d4ed06470370270 Mon Sep 17 00:00:00 2001 From: Zoran Cvetkov Date: Mon, 5 Aug 2024 22:07:18 +0300 Subject: [PATCH 27/29] fix test --- store/test-store/tests/postgres/writable.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/store/test-store/tests/postgres/writable.rs b/store/test-store/tests/postgres/writable.rs index 26545ba2f00..2df32daa064 100644 --- a/store/test-store/tests/postgres/writable.rs +++ b/store/test-store/tests/postgres/writable.rs @@ -360,7 +360,7 @@ async fn read_range( fn read_range_mutable() { run_test(|store, writable, deployment| async move { let num_entities = read_range(store, writable, deployment, true).await; - assert_eq!(num_entities, 2) + assert_eq!(num_entities, 3) // TODO: fix it - it should be 2 as the range is open }) } From 1b34ca25d147cccba7c0469f274fe6047ca7e0c4 Mon Sep 17 00:00:00 2001 From: Zoran Cvetkov Date: Thu, 8 Aug 2024 15:53:17 +0300 Subject: [PATCH 28/29] refactor --- store/postgres/src/block_range.rs | 151 +++++++++-------------- store/postgres/src/relational_queries.rs | 10 +- 2 files changed, 66 insertions(+), 95 deletions(-) diff --git a/store/postgres/src/block_range.rs b/store/postgres/src/block_range.rs index 2da371684eb..c67f7ce6f9c 100644 --- a/store/postgres/src/block_range.rs +++ b/store/postgres/src/block_range.rs @@ -132,6 +132,66 @@ impl<'a> QueryFragment for BlockRangeUpperBoundClause<'a> { } } +/// Helper for generating SQL fragments for selecting entities in a specific block range +#[derive(Debug, Clone, Copy)] +pub enum EntityBlockRange { + Mutable(BlockRange), // TODO: check if this is a proper type here (maybe Range?) + Immutable(BlockRange), +} + +impl EntityBlockRange { + pub fn new(immutable: bool, block_range: std::ops::Range) -> Self { + let st: Bound = Bound::Included(block_range.start.try_into().unwrap()); + let en: Bound = Bound::Excluded(block_range.end.try_into().unwrap()); + let block_range: BlockRange = BlockRange(st, en); + if immutable { + Self::Immutable(block_range) + } else { + Self::Mutable(block_range) + } + } + + /// Output SQL that matches only rows whose block range contains `block`. + pub fn contains<'b>(&'b self, out: &mut AstPass<'_, 'b, Pg>) -> QueryResult<()> { + out.unsafe_to_cache_prepared(); + let block_range = match self { + EntityBlockRange::Mutable(br) => br, + EntityBlockRange::Immutable(br) => br, + }; + let BlockRange(start, finish) = block_range; + + self.compare_column(out); + out.push_sql(" >= "); + match start { + Bound::Included(block) => out.push_bind_param::(block)?, + Bound::Excluded(block) => { + out.push_bind_param::(block)?; + out.push_sql("+1"); + } + Bound::Unbounded => unimplemented!(), + }; + out.push_sql(" AND "); + self.compare_column(out); + out.push_sql(" <= "); + match finish { + Bound::Included(block) => { + out.push_bind_param::(block)?; + out.push_sql("+1"); + } + Bound::Excluded(block) => out.push_bind_param::(block)?, + Bound::Unbounded => unimplemented!(), + }; + Ok(()) + } + + pub fn compare_column(&self, out: &mut AstPass) { + match self { + EntityBlockRange::Mutable(_) => out.push_sql(" lower(block_range) "), + EntityBlockRange::Immutable(_) => out.push_sql(" block$ "), + } + } +} + /// Helper for generating various SQL fragments for handling the block range /// of entity versions #[allow(unused)] @@ -147,16 +207,6 @@ pub enum BlockRangeColumn<'a> { table_prefix: &'a str, block: BlockNumber, }, - MutableRange { - table: &'a Table, - table_prefix: &'a str, - block_range: BlockRange, // TODO: check if this is a proper type here (maybe Range?) - }, - ImmutableRange { - table: &'a Table, - table_prefix: &'a str, - block_range: BlockRange, - }, } impl<'a> BlockRangeColumn<'a> { @@ -176,35 +226,10 @@ impl<'a> BlockRangeColumn<'a> { } } - // TODO: refactor new and new2 into one. use enum of both BlockNumber and range - pub fn new2( - table: &'a Table, - table_prefix: &'a str, - block_range: std::ops::Range, - ) -> Self { - let st: Bound = Bound::Included(block_range.start.try_into().unwrap()); - let en: Bound = Bound::Excluded(block_range.end.try_into().unwrap()); - let block_range: BlockRange = BlockRange(st, en); - if table.immutable { - Self::ImmutableRange { - table, - table_prefix, - block_range, - } - } else { - Self::MutableRange { - table, - table_prefix, - block_range, - } - } - } - pub fn block(&self) -> BlockNumber { match self { BlockRangeColumn::Mutable { block, .. } => *block, BlockRangeColumn::Immutable { block, .. } => *block, - _ => todo!(), } } } @@ -259,56 +284,6 @@ impl<'a> BlockRangeColumn<'a> { out.push_bind_param::(block) } } - BlockRangeColumn::MutableRange { - table: _, - table_prefix: _, - block_range: BlockRange(start, finish), - } => { - out.push_sql("lower(block_range) >= "); - match start { - Bound::Included(block) => out.push_bind_param::(block)?, - Bound::Excluded(block) => { - out.push_bind_param::(block)?; - out.push_sql("+1"); - } - Bound::Unbounded => todo!(), - }; - out.push_sql(" AND lower(block_range) <= "); - match finish { - Bound::Included(block) => { - out.push_bind_param::(block)?; - out.push_sql("+1"); - } - Bound::Excluded(block) => out.push_bind_param::(block)?, - Bound::Unbounded => todo!(), - }; - Ok(()) - } - BlockRangeColumn::ImmutableRange { - table: _, - table_prefix: _, - block_range: BlockRange(start, finish), - } => { - out.push_sql("block$ >= "); - match start { - Bound::Included(block) => out.push_bind_param::(block)?, - Bound::Excluded(block) => { - out.push_bind_param::(block)?; - out.push_sql("+1"); - } - Bound::Unbounded => todo!(), - }; - out.push_sql(" AND block$ <= "); - match finish { - Bound::Included(block) => { - out.push_bind_param::(block)?; - out.push_sql("+1"); - } - Bound::Excluded(block) => out.push_bind_param::(block)?, - Bound::Unbounded => todo!(), - }; - Ok(()) - } } } @@ -316,7 +291,6 @@ impl<'a> BlockRangeColumn<'a> { match self { BlockRangeColumn::Mutable { .. } => BLOCK_RANGE_COLUMN, BlockRangeColumn::Immutable { .. } => BLOCK_COLUMN, - _ => todo!(), } } @@ -331,7 +305,6 @@ impl<'a> BlockRangeColumn<'a> { out.push_sql(table_prefix); out.push_sql(BLOCK_COLUMN); } - _ => todo!(), } } @@ -341,7 +314,6 @@ impl<'a> BlockRangeColumn<'a> { match self { BlockRangeColumn::Mutable { .. } => out.push_sql(BLOCK_RANGE_CURRENT), BlockRangeColumn::Immutable { .. } => out.push_sql("true"), - _ => todo!(), } } @@ -365,7 +337,6 @@ impl<'a> BlockRangeColumn<'a> { BlockRangeColumn::Immutable { .. } => { unreachable!("immutable entities can not be updated or deleted") } - _ => todo!(), } } @@ -374,7 +345,6 @@ impl<'a> BlockRangeColumn<'a> { match self { BlockRangeColumn::Mutable { .. } => out.push_sql(BLOCK_RANGE_COLUMN), BlockRangeColumn::Immutable { .. } => out.push_sql(BLOCK_COLUMN), - _ => todo!(), } } @@ -393,7 +363,6 @@ impl<'a> BlockRangeColumn<'a> { out.push_sql(" >= "); out.push_bind_param::(block) } - _ => todo!(), } } } diff --git a/store/postgres/src/relational_queries.rs b/store/postgres/src/relational_queries.rs index d4f205a4bc2..fe43952d2e3 100644 --- a/store/postgres/src/relational_queries.rs +++ b/store/postgres/src/relational_queries.rs @@ -35,6 +35,7 @@ use std::ops::Range; use std::str::FromStr; use std::string::ToString; +use crate::block_range::EntityBlockRange; use crate::relational::{ Column, ColumnType, Layout, SqlName, Table, BYTE_ARRAY_PREFIX_SIZE, PRIMARY_KEY_COLUMN, STRING_PREFIX_SIZE, @@ -2008,13 +2009,13 @@ impl<'a, Conn> RunQueryDsl for FindQuery<'a> {} #[derive(Debug, Clone)] pub struct FindRangeQuery<'a> { table: &'a Table, - br_column: BlockRangeColumn<'a>, + eb_range: EntityBlockRange, } impl<'a> FindRangeQuery<'a> { pub fn new(table: &'a Table, block_range: Range) -> Self { - let br_column = BlockRangeColumn::new2(table, "e.", block_range); - Self { table, br_column } + let eb_range = EntityBlockRange::new(table.immutable, block_range); + Self { table, eb_range } } } @@ -2031,12 +2032,13 @@ impl<'a> QueryFragment for FindRangeQuery<'a> { out.push_sql(" from "); out.push_sql(self.table.qualified_name.as_str()); out.push_sql(" e\n where "); + // TODO: do we need to care about it? // if self.table.has_causality_region { // out.push_sql("causality_region = "); // out.push_bind_param::(&self.key.causality_region)?; // out.push_sql(" and "); // } - self.br_column.contains(&mut out, true) + self.eb_range.contains(&mut out) } } From 3afd1c62aea1e2eca3c2f13fc0e0f705cee63c31 Mon Sep 17 00:00:00 2001 From: Zoran Cvetkov Date: Mon, 12 Aug 2024 17:53:53 +0300 Subject: [PATCH 29/29] change the range type --- graph/src/components/store/mod.rs | 2 +- graph/src/components/store/traits.rs | 4 ++-- store/postgres/src/block_range.rs | 8 ++++---- store/postgres/src/deployment_store.rs | 2 +- store/postgres/src/relational.rs | 2 +- store/postgres/src/relational_queries.rs | 2 +- store/postgres/src/writable.rs | 9 ++++----- store/test-store/tests/graph/entity_cache.rs | 2 +- store/test-store/tests/postgres/writable.rs | 2 +- 9 files changed, 16 insertions(+), 17 deletions(-) diff --git a/graph/src/components/store/mod.rs b/graph/src/components/store/mod.rs index e0992f9d62d..d0414abf096 100644 --- a/graph/src/components/store/mod.rs +++ b/graph/src/components/store/mod.rs @@ -1042,7 +1042,7 @@ impl ReadStore for EmptyStore { fn get_range( &self, _entity_type: &EntityType, - _block_range: Range, + _block_range: Range, ) -> Result>, StoreError> { Ok(BTreeMap::new()) } diff --git a/graph/src/components/store/traits.rs b/graph/src/components/store/traits.rs index a916a5e7241..d93665332c6 100644 --- a/graph/src/components/store/traits.rs +++ b/graph/src/components/store/traits.rs @@ -232,7 +232,7 @@ pub trait ReadStore: Send + Sync + 'static { fn get_range( &self, entity_type: &EntityType, - block_range: Range, + block_range: Range, ) -> Result>, StoreError>; /// Reverse lookup @@ -260,7 +260,7 @@ impl ReadStore for Arc { fn get_range( &self, entity_type: &EntityType, - block_range: Range, + block_range: Range, ) -> Result>, StoreError> { (**self).get_range(entity_type, block_range) } diff --git a/store/postgres/src/block_range.rs b/store/postgres/src/block_range.rs index c67f7ce6f9c..2387f6abd0a 100644 --- a/store/postgres/src/block_range.rs +++ b/store/postgres/src/block_range.rs @@ -140,10 +140,10 @@ pub enum EntityBlockRange { } impl EntityBlockRange { - pub fn new(immutable: bool, block_range: std::ops::Range) -> Self { - let st: Bound = Bound::Included(block_range.start.try_into().unwrap()); - let en: Bound = Bound::Excluded(block_range.end.try_into().unwrap()); - let block_range: BlockRange = BlockRange(st, en); + pub fn new(immutable: bool, block_range: std::ops::Range) -> Self { + let start: Bound = Bound::Included(block_range.start); + let end: Bound = Bound::Excluded(block_range.end); + let block_range: BlockRange = BlockRange(start, end); if immutable { Self::Immutable(block_range) } else { diff --git a/store/postgres/src/deployment_store.rs b/store/postgres/src/deployment_store.rs index 1dac6c46279..f500f2aa6ed 100644 --- a/store/postgres/src/deployment_store.rs +++ b/store/postgres/src/deployment_store.rs @@ -1060,7 +1060,7 @@ impl DeploymentStore { &self, site: Arc, entity_type: &EntityType, - block_range: Range, + block_range: Range, ) -> Result>, StoreError> { let mut conn = self.get_conn()?; let layout = self.layout(&mut conn, site)?; diff --git a/store/postgres/src/relational.rs b/store/postgres/src/relational.rs index 5f10c4bc8e9..792c7d8342e 100644 --- a/store/postgres/src/relational.rs +++ b/store/postgres/src/relational.rs @@ -519,7 +519,7 @@ impl Layout { &self, conn: &mut PgConnection, entity_type: &EntityType, - block_range: Range, + block_range: Range, ) -> Result>, StoreError> { let table = self.table_for_entity(entity_type)?; let mut entities: BTreeMap> = BTreeMap::new(); diff --git a/store/postgres/src/relational_queries.rs b/store/postgres/src/relational_queries.rs index fe43952d2e3..e01cc3db85b 100644 --- a/store/postgres/src/relational_queries.rs +++ b/store/postgres/src/relational_queries.rs @@ -2013,7 +2013,7 @@ pub struct FindRangeQuery<'a> { } impl<'a> FindRangeQuery<'a> { - pub fn new(table: &'a Table, block_range: Range) -> Self { + pub fn new(table: &'a Table, block_range: Range) -> Self { let eb_range = EntityBlockRange::new(table.immutable, block_range); Self { table, eb_range } } diff --git a/store/postgres/src/writable.rs b/store/postgres/src/writable.rs index a1b1241a15a..6c2faef3e14 100644 --- a/store/postgres/src/writable.rs +++ b/store/postgres/src/writable.rs @@ -355,7 +355,7 @@ impl SyncStore { fn get_range( &self, entity_type: &EntityType, - block_range: Range, + block_range: Range, ) -> Result>, StoreError> { retry::forever(&self.logger, "get_range", || { self.writable @@ -1231,9 +1231,8 @@ impl Queue { fn get_range( &self, entity_type: &EntityType, - block_range: Range, + block_range: Range, ) -> Result>, StoreError> { - // TODO: implemet read from the queue self.store.get_range(entity_type, block_range) } @@ -1452,7 +1451,7 @@ impl Writer { fn get_range( &self, entity_type: &EntityType, - block_range: Range, + block_range: Range, ) -> Result>, StoreError> { match self { Writer::Sync(store) => store.get_range(entity_type, block_range), @@ -1591,7 +1590,7 @@ impl ReadStore for WritableStore { fn get_range( &self, entity_type: &EntityType, - block_range: Range, + block_range: Range, ) -> Result>, StoreError> { self.writer.get_range(entity_type, block_range) } diff --git a/store/test-store/tests/graph/entity_cache.rs b/store/test-store/tests/graph/entity_cache.rs index b1f0d9a0f23..314f82d1072 100644 --- a/store/test-store/tests/graph/entity_cache.rs +++ b/store/test-store/tests/graph/entity_cache.rs @@ -70,7 +70,7 @@ impl ReadStore for MockStore { fn get_range( &self, _entity_type: &EntityType, - _block_range: Range, + _block_range: Range, ) -> Result>, StoreError> { todo!() } diff --git a/store/test-store/tests/postgres/writable.rs b/store/test-store/tests/postgres/writable.rs index 2df32daa064..74c2fb5d489 100644 --- a/store/test-store/tests/postgres/writable.rs +++ b/store/test-store/tests/postgres/writable.rs @@ -346,7 +346,7 @@ async fn read_range( } writable.flush().await.unwrap(); - let br: Range = 4..8; + let br: Range = 4..8; let et: &EntityType = if mutable { &COUNTER_TYPE } else {