From 605eb530b60057966dd5f318e1ef91e3c9dde9ff Mon Sep 17 00:00:00 2001 From: shiyasmohd Date: Mon, 30 Dec 2024 22:03:02 +0530 Subject: [PATCH 1/9] core,node: move config to core, use config check(cli) from core & update imports --- Cargo.lock | 8 +++- core/graphman/Cargo.toml | 4 ++ core/graphman/src/commands/config/check.rs | 42 +++++++++++++++++++ core/graphman/src/commands/config/mod.rs | 1 + core/graphman/src/commands/mod.rs | 1 + {node => core/graphman}/src/config.rs | 2 +- core/graphman/src/lib.rs | 1 + node/src/bin/manager.rs | 4 +- node/src/chain.rs | 6 +-- node/src/lib.rs | 1 - node/src/main.rs | 2 +- node/src/manager/commands/config.rs | 3 +- node/src/manager/commands/config_cmd/check.rs | 26 ++++++++++++ node/src/manager/commands/config_cmd/mod.rs | 1 + node/src/manager/commands/mod.rs | 1 + node/src/manager/commands/run.rs | 2 +- node/src/network_setup.rs | 2 +- node/src/opt.rs | 2 +- node/src/store_builder.rs | 2 +- store/test-store/Cargo.toml | 1 + store/test-store/src/store.rs | 2 +- tests/Cargo.toml | 8 +++- tests/src/fixture/mod.rs | 3 +- 23 files changed, 107 insertions(+), 18 deletions(-) create mode 100644 core/graphman/src/commands/config/check.rs create mode 100644 core/graphman/src/commands/config/mod.rs rename {node => core/graphman}/src/config.rs (100%) create mode 100644 node/src/manager/commands/config_cmd/check.rs create mode 100644 node/src/manager/commands/config_cmd/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 17573acd02c..18d99946109 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "Inflector" @@ -2183,6 +2183,7 @@ dependencies = [ "graph-runtime-wasm", "graph-server-index-node", "graph-store-postgres", + "graphman", "secp256k1", "serde", "serde_yaml", @@ -2209,11 +2210,15 @@ dependencies = [ "anyhow", "diesel", "graph", + "graph-chain-ethereum", "graph-store-postgres", "graphman-store", "itertools 0.13.0", + "serde", + "shellexpand", "thiserror", "tokio", + "url", ] [[package]] @@ -5034,6 +5039,7 @@ dependencies = [ "graph-graphql", "graph-node", "graph-store-postgres", + "graphman", "hex", "hex-literal 0.4.1", "lazy_static", diff --git a/core/graphman/Cargo.toml b/core/graphman/Cargo.toml index 001a683f4aa..b39515ccb78 100644 --- a/core/graphman/Cargo.toml +++ b/core/graphman/Cargo.toml @@ -12,3 +12,7 @@ graphman-store = { workspace = true } itertools = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } +graph-chain-ethereum = { path = "../../chain/ethereum" } +serde = { workspace = true } +url = "2.5.2" +shellexpand = "3.1.0" diff --git a/core/graphman/src/commands/config/check.rs b/core/graphman/src/commands/config/check.rs new file mode 100644 index 00000000000..5f9831b83f4 --- /dev/null +++ b/core/graphman/src/commands/config/check.rs @@ -0,0 +1,42 @@ +use crate::config::Config; +use anyhow::Error; +use graph::{components::subgraph::Settings, env::EnvVars}; + +pub struct ConfigCheckResult { + pub validated: bool, + pub validated_subgraph_settings: bool, + pub config_json: Option, +} + +pub fn check(config: &Config, print: bool) -> Result { + let mut result = ConfigCheckResult { + validated: false, + validated_subgraph_settings: false, + config_json: None, + }; + if print { + match config.to_json() { + Ok(txt) => { + result.config_json = Some(txt); + } + Err(err) => return Err(anyhow::format_err!("error serializing config: {}", err)), + }; + } + let env_vars = EnvVars::from_env().unwrap(); + if let Some(path) = &env_vars.subgraph_settings { + match Settings::from_file(path) { + Ok(_) => { + result.validated_subgraph_settings = true; + } + Err(e) => { + return Err(anyhow::format_err!( + "configuration error in subgraph settings {}: {}", + path, + e + )); + } + } + }; + result.validated = true; + Ok(result) +} diff --git a/core/graphman/src/commands/config/mod.rs b/core/graphman/src/commands/config/mod.rs new file mode 100644 index 00000000000..3e8ff0fc6bd --- /dev/null +++ b/core/graphman/src/commands/config/mod.rs @@ -0,0 +1 @@ +pub mod check; diff --git a/core/graphman/src/commands/mod.rs b/core/graphman/src/commands/mod.rs index 98629027b58..ffa528041a0 100644 --- a/core/graphman/src/commands/mod.rs +++ b/core/graphman/src/commands/mod.rs @@ -1 +1,2 @@ +pub mod config; pub mod deployment; diff --git a/node/src/config.rs b/core/graphman/src/config.rs similarity index 100% rename from node/src/config.rs rename to core/graphman/src/config.rs index 8006b8efef7..d486f69ae8f 100644 --- a/node/src/config.rs +++ b/core/graphman/src/config.rs @@ -19,9 +19,9 @@ use graph::{ use graph_chain_ethereum as ethereum; use graph_chain_ethereum::NodeCapabilities; use graph_store_postgres::{DeploymentPlacer, Shard as ShardName, PRIMARY_SHARD}; +use serde::Serialize; use graph::http::{HeaderMap, Uri}; -use serde::Serialize; use std::{ collections::{BTreeMap, BTreeSet}, fmt, diff --git a/core/graphman/src/lib.rs b/core/graphman/src/lib.rs index 71f8e77a848..83848a78e83 100644 --- a/core/graphman/src/lib.rs +++ b/core/graphman/src/lib.rs @@ -8,6 +8,7 @@ mod error; pub mod commands; +pub mod config; pub mod deployment; pub mod execution_tracker; diff --git a/node/src/bin/manager.rs b/node/src/bin/manager.rs index afc09403357..a466c72f364 100644 --- a/node/src/bin/manager.rs +++ b/node/src/bin/manager.rs @@ -19,7 +19,6 @@ use graph::{ }; use graph_chain_ethereum::EthereumAdapter; use graph_graphql::prelude::GraphQlRunner; -use graph_node::config::{self, Config as Cfg}; use graph_node::manager::color::Terminal; use graph_node::manager::commands; use graph_node::network_setup::Networks; @@ -34,6 +33,7 @@ use graph_store_postgres::{ connection_pool::ConnectionPool, BlockStore, NotificationSender, Shard, Store, SubgraphStore, SubscriptionManager, PRIMARY_SHARD, }; +use graphman::config::{self, Config as Cfg}; use itertools::Itertools; use lazy_static::lazy_static; use std::str::FromStr; @@ -1184,7 +1184,7 @@ async fn main() -> anyhow::Result<()> { Place { name, network } => { commands::config::place(&ctx.config.deployment, &name, &network) } - Check { print } => commands::config::check(&ctx.config, print), + Check { print } => commands::config_cmd::check::run(&ctx.config, print), Pools { nodes, shard } => commands::config::pools(&ctx.config, nodes, shard), Provider { features, network } => { let logger = ctx.logger.clone(); diff --git a/node/src/chain.rs b/node/src/chain.rs index 1c62bf2248e..332b8367c87 100644 --- a/node/src/chain.rs +++ b/node/src/chain.rs @@ -1,4 +1,3 @@ -use crate::config::{Config, ProviderDetails}; use crate::network_setup::{ AdapterConfiguration, EthAdapterConfig, FirehoseAdapterConfig, Networks, }; @@ -31,6 +30,7 @@ use graph::tokio::time::timeout; use graph::url::Url; use graph_chain_ethereum::{self as ethereum, Transport}; use graph_store_postgres::{BlockStore, ChainHeadUpdateListener}; +use graphman::config::{Config, ProviderDetails}; use std::cmp::Ordering; use std::collections::BTreeMap; use std::sync::Arc; @@ -240,7 +240,7 @@ pub async fn create_ethereum_networks_for_chain( "capabilities" => capabilities ); - use crate::config::Transport::*; + use graphman::config::Transport::*; let transport = match web3.transport { Rpc => Transport::new_rpc( @@ -553,13 +553,13 @@ pub async fn networks_as_chains( #[cfg(test)] mod test { - use crate::config::{Config, Opt}; use crate::network_setup::{AdapterConfiguration, Networks}; use graph::components::network_provider::ChainName; use graph::endpoint::EndpointMetrics; use graph::log::logger; use graph::prelude::{tokio, MetricsRegistry}; use graph_chain_ethereum::NodeCapabilities; + use graphman::config::{Config, Opt}; use std::sync::Arc; #[tokio::test] diff --git a/node/src/lib.rs b/node/src/lib.rs index f65ffc1be8f..b03bef2f13d 100644 --- a/node/src/lib.rs +++ b/node/src/lib.rs @@ -6,7 +6,6 @@ use graph::{prelude::MetricsRegistry, prometheus::Registry}; extern crate diesel; pub mod chain; -pub mod config; pub mod network_setup; pub mod opt; pub mod store_builder; diff --git a/node/src/main.rs b/node/src/main.rs index 870cce97318..5a7591a85e3 100644 --- a/node/src/main.rs +++ b/node/src/main.rs @@ -20,7 +20,6 @@ use graph_core::{ SubgraphRegistrar as IpfsSubgraphRegistrar, }; use graph_graphql::prelude::GraphQlRunner; -use graph_node::config::Config; use graph_node::network_setup::Networks; use graph_node::opt; use graph_node::store_builder::StoreBuilder; @@ -32,6 +31,7 @@ use graph_server_websocket::SubscriptionServer as GraphQLSubscriptionServer; use graph_store_postgres::connection_pool::ConnectionPool; use graph_store_postgres::Store; use graph_store_postgres::{register_jobs as register_store_jobs, NotificationSender}; +use graphman::config::Config; use graphman_server::GraphmanServer; use graphman_server::GraphmanServerConfig; use std::io::{BufRead, BufReader}; diff --git a/node/src/manager/commands/config.rs b/node/src/manager/commands/config.rs index 8b6d36e9afa..8c29641e1a0 100644 --- a/node/src/manager/commands/config.rs +++ b/node/src/manager/commands/config.rs @@ -16,7 +16,8 @@ use graph::{ use graph_chain_ethereum::NodeCapabilities; use graph_store_postgres::DeploymentPlacer; -use crate::{config::Config, network_setup::Networks}; +use crate::network_setup::Networks; +use graphman::config::Config; pub fn place(placer: &dyn DeploymentPlacer, name: &str, network: &str) -> Result<(), Error> { match placer.place(name, network).map_err(|s| anyhow!(s))? { diff --git a/node/src/manager/commands/config_cmd/check.rs b/node/src/manager/commands/config_cmd/check.rs new file mode 100644 index 00000000000..abd3563ea6a --- /dev/null +++ b/node/src/manager/commands/config_cmd/check.rs @@ -0,0 +1,26 @@ +use anyhow::Error; +use graphman::{commands::config::check::check, config::Config}; + +pub fn run(config: &Config, print: bool) -> Result<(), Error> { + let check = check(config, print); + + match check { + Ok(res) => { + if res.validated { + println!("Successfully validated configuration"); + } + if res.validated_subgraph_settings { + println!("Successfully validated subgraph settings"); + } + if let Some(txt) = res.config_json { + println!("{}", txt); + } + } + Err(e) => { + eprintln!("configuration error: {}", e); + std::process::exit(1); + } + } + + Ok(()) +} diff --git a/node/src/manager/commands/config_cmd/mod.rs b/node/src/manager/commands/config_cmd/mod.rs new file mode 100644 index 00000000000..3e8ff0fc6bd --- /dev/null +++ b/node/src/manager/commands/config_cmd/mod.rs @@ -0,0 +1 @@ +pub mod check; diff --git a/node/src/manager/commands/mod.rs b/node/src/manager/commands/mod.rs index cb81a19ecb3..d15128b07e9 100644 --- a/node/src/manager/commands/mod.rs +++ b/node/src/manager/commands/mod.rs @@ -2,6 +2,7 @@ pub mod assign; pub mod chain; pub mod check_blocks; pub mod config; +pub mod config_cmd; pub mod copy; pub mod create; pub mod database; diff --git a/node/src/manager/commands/run.rs b/node/src/manager/commands/run.rs index 2c6bfdcb148..e724bd99a75 100644 --- a/node/src/manager/commands/run.rs +++ b/node/src/manager/commands/run.rs @@ -2,7 +2,6 @@ use std::collections::HashMap; use std::sync::Arc; use std::time::Duration; -use crate::config::Config; use crate::manager::PanicSubscriptionManager; use crate::network_setup::Networks; use crate::store_builder::StoreBuilder; @@ -26,6 +25,7 @@ use graph_core::{ SubgraphAssignmentProvider as IpfsSubgraphAssignmentProvider, SubgraphInstanceManager, SubgraphRegistrar as IpfsSubgraphRegistrar, }; +use graphman::config::Config; fn locate(store: &dyn SubgraphStore, hash: &str) -> Result { let mut locators = store.locators(hash)?; diff --git a/node/src/network_setup.rs b/node/src/network_setup.rs index 4a8b4cedca1..213c0a4bd5b 100644 --- a/node/src/network_setup.rs +++ b/node/src/network_setup.rs @@ -185,7 +185,7 @@ impl Networks { pub async fn from_config( logger: Logger, - config: &crate::config::Config, + config: &graphman::config::Config, registry: Arc, endpoint_metrics: Arc, provider_checks: &[Arc], diff --git a/node/src/opt.rs b/node/src/opt.rs index e4dc44ba92a..62c60286843 100644 --- a/node/src/opt.rs +++ b/node/src/opt.rs @@ -2,7 +2,7 @@ use clap::Parser; use git_testament::{git_testament, render_testament}; use lazy_static::lazy_static; -use crate::config; +use graphman::config; git_testament!(TESTAMENT); lazy_static! { diff --git a/node/src/store_builder.rs b/node/src/store_builder.rs index 2a39d0ea6ed..9deb1b72fc7 100644 --- a/node/src/store_builder.rs +++ b/node/src/store_builder.rs @@ -17,7 +17,7 @@ use graph_store_postgres::{ SubscriptionManager, PRIMARY_SHARD, }; -use crate::config::{Config, Shard}; +use graphman::config::{Config, Shard}; pub struct StoreBuilder { logger: Logger, diff --git a/store/test-store/Cargo.toml b/store/test-store/Cargo.toml index fe05f12233e..89dcd0ce25c 100644 --- a/store/test-store/Cargo.toml +++ b/store/test-store/Cargo.toml @@ -15,6 +15,7 @@ lazy_static = "1.5" hex-literal = "0.4" diesel = { workspace = true } prost-types = { workspace = true } +graphman = { workspace = true } [dev-dependencies] hex = "0.4.3" diff --git a/store/test-store/src/store.rs b/store/test-store/src/store.rs index afb088f6bf6..ad289841ec3 100644 --- a/store/test-store/src/store.rs +++ b/store/test-store/src/store.rs @@ -23,7 +23,6 @@ use graph_graphql::prelude::{ execute_query, Query as PreparedQuery, QueryExecutionOptions, StoreResolver, }; use graph_graphql::test_support::GraphQLMetrics; -use graph_node::config::{Config, Opt}; use graph_node::store_builder::StoreBuilder; use graph_store_postgres::layout_for_tests::FAKE_NETWORK_SHARED; use graph_store_postgres::{connection_pool::ConnectionPool, Shard, SubscriptionManager}; @@ -31,6 +30,7 @@ use graph_store_postgres::{ BlockStore as DieselBlockStore, DeploymentPlacer, SubgraphStore as DieselSubgraphStore, PRIMARY_SHARD, }; +use graphman::config::{Config, Opt}; use hex_literal::hex; use lazy_static::lazy_static; use std::collections::BTreeSet; diff --git a/tests/Cargo.toml b/tests/Cargo.toml index b79702ceb7f..f350b295d75 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -9,16 +9,20 @@ assert-json-diff = "2.0.2" async-stream = "0.3.5" graph = { path = "../graph" } graph-chain-ethereum = { path = "../chain/ethereum" } -graph-chain-substreams= {path = "../chain/substreams"} +graph-chain-substreams = { path = "../chain/substreams" } graph-node = { path = "../node" } graph-core = { path = "../core" } graph-graphql = { path = "../graphql" } graph-store-postgres = { path = "../store/postgres" } graph-server-index-node = { path = "../server/index-node" } graph-runtime-wasm = { path = "../runtime/wasm" } +graphman = { workspace = true } serde = { workspace = true } serde_yaml = { workspace = true } -slog = { version = "2.7.0", features = ["release_max_level_trace", "max_level_trace"] } +slog = { version = "2.7.0", features = [ + "release_max_level_trace", + "max_level_trace", +] } tokio = { version = "1.38.0", features = ["rt", "macros", "process"] } # Once graph upgrades to web3 0.19, we don't need this anymore. The version # here needs to be kept in sync with the web3 version that the graph crate diff --git a/tests/src/fixture/mod.rs b/tests/src/fixture/mod.rs index 4eb5fbb42b1..d1d021fc9c2 100644 --- a/tests/src/fixture/mod.rs +++ b/tests/src/fixture/mod.rs @@ -51,10 +51,11 @@ use graph_core::{ SubgraphRegistrar as IpfsSubgraphRegistrar, SubgraphTriggerProcessor, }; use graph_node::manager::PanicSubscriptionManager; -use graph_node::{config::Config, store_builder::StoreBuilder}; +use graph_node::store_builder::StoreBuilder; use graph_runtime_wasm::RuntimeHostBuilder; use graph_server_index_node::IndexNodeService; use graph_store_postgres::{ChainHeadUpdateListener, ChainStore, Store, SubgraphStore}; +use graphman::config::Config; use serde::Deserialize; use slog::{crit, debug, info, o, Discard, Logger}; use std::env::VarError; From eab550d155b110981827013b56c9b625fc9ec456 Mon Sep 17 00:00:00 2001 From: shiyasmohd Date: Wed, 1 Jan 2025 15:25:06 +0530 Subject: [PATCH 2/9] core,node,server: move opt to core and add graphman config check to graphql api --- Cargo.lock | 5 +++++ core/graphman/Cargo.toml | 4 ++++ core/graphman/src/lib.rs | 1 + {node => core/graphman}/src/opt.rs | 2 +- node/src/lib.rs | 1 - node/src/main.rs | 2 +- server/graphman/Cargo.toml | 1 + server/graphman/src/entities/config_check.rs | 21 +++++++++++++++++++ server/graphman/src/entities/mod.rs | 2 ++ server/graphman/src/resolvers/config_query.rs | 16 ++++++++++++++ .../src/resolvers/config_query/check.rs | 14 +++++++++++++ server/graphman/src/resolvers/context.rs | 16 ++++++++++++++ server/graphman/src/resolvers/mod.rs | 2 ++ server/graphman/src/resolvers/query_root.rs | 6 ++++++ 14 files changed, 90 insertions(+), 3 deletions(-) rename {node => core/graphman}/src/opt.rs (99%) create mode 100644 server/graphman/src/entities/config_check.rs create mode 100644 server/graphman/src/resolvers/config_query.rs create mode 100644 server/graphman/src/resolvers/config_query/check.rs diff --git a/Cargo.lock b/Cargo.lock index 18d99946109..3341661f11c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2208,12 +2208,16 @@ name = "graphman" version = "0.36.0" dependencies = [ "anyhow", + "async-graphql", + "clap", "diesel", + "git-testament", "graph", "graph-chain-ethereum", "graph-store-postgres", "graphman-store", "itertools 0.13.0", + "lazy_static", "serde", "shellexpand", "thiserror", @@ -2230,6 +2234,7 @@ dependencies = [ "async-graphql-axum", "axum 0.7.5", "chrono", + "clap", "diesel", "graph", "graph-store-postgres", diff --git a/core/graphman/Cargo.toml b/core/graphman/Cargo.toml index b39515ccb78..c25d9ffadc2 100644 --- a/core/graphman/Cargo.toml +++ b/core/graphman/Cargo.toml @@ -5,11 +5,15 @@ edition.workspace = true [dependencies] anyhow = { workspace = true } +async-graphql = { workspace = true } +clap = { workspace = true } diesel = { workspace = true } +git-testament = "0.2" graph = { workspace = true } graph-store-postgres = { workspace = true } graphman-store = { workspace = true } itertools = { workspace = true } +lazy_static = "1.5.0" thiserror = { workspace = true } tokio = { workspace = true } graph-chain-ethereum = { path = "../../chain/ethereum" } diff --git a/core/graphman/src/lib.rs b/core/graphman/src/lib.rs index 83848a78e83..d48bf9bbb44 100644 --- a/core/graphman/src/lib.rs +++ b/core/graphman/src/lib.rs @@ -11,6 +11,7 @@ pub mod commands; pub mod config; pub mod deployment; pub mod execution_tracker; +pub mod opt; pub use self::error::GraphmanError; pub use self::execution_tracker::GraphmanExecutionTracker; diff --git a/node/src/opt.rs b/core/graphman/src/opt.rs similarity index 99% rename from node/src/opt.rs rename to core/graphman/src/opt.rs index 62c60286843..e4dc44ba92a 100644 --- a/node/src/opt.rs +++ b/core/graphman/src/opt.rs @@ -2,7 +2,7 @@ use clap::Parser; use git_testament::{git_testament, render_testament}; use lazy_static::lazy_static; -use graphman::config; +use crate::config; git_testament!(TESTAMENT); lazy_static! { diff --git a/node/src/lib.rs b/node/src/lib.rs index b03bef2f13d..7f78083f1fa 100644 --- a/node/src/lib.rs +++ b/node/src/lib.rs @@ -7,7 +7,6 @@ extern crate diesel; pub mod chain; pub mod network_setup; -pub mod opt; pub mod store_builder; pub mod manager; diff --git a/node/src/main.rs b/node/src/main.rs index 5a7591a85e3..ada0f53876c 100644 --- a/node/src/main.rs +++ b/node/src/main.rs @@ -21,7 +21,6 @@ use graph_core::{ }; use graph_graphql::prelude::GraphQlRunner; use graph_node::network_setup::Networks; -use graph_node::opt; use graph_node::store_builder::StoreBuilder; use graph_server_http::GraphQLServer as GraphQLQueryServer; use graph_server_index_node::IndexNodeServer; @@ -32,6 +31,7 @@ use graph_store_postgres::connection_pool::ConnectionPool; use graph_store_postgres::Store; use graph_store_postgres::{register_jobs as register_store_jobs, NotificationSender}; use graphman::config::Config; +use graphman::opt; use graphman_server::GraphmanServer; use graphman_server::GraphmanServerConfig; use std::io::{BufRead, BufReader}; diff --git a/server/graphman/Cargo.toml b/server/graphman/Cargo.toml index 231ef5e0828..ab8313bc4a3 100644 --- a/server/graphman/Cargo.toml +++ b/server/graphman/Cargo.toml @@ -9,6 +9,7 @@ async-graphql = { workspace = true } async-graphql-axum = { workspace = true } axum = { workspace = true } chrono = { workspace = true } +clap = { workspace = true } graph = { workspace = true } graph-store-postgres = { workspace = true } graphman = { workspace = true } diff --git a/server/graphman/src/entities/config_check.rs b/server/graphman/src/entities/config_check.rs new file mode 100644 index 00000000000..e25b32d6844 --- /dev/null +++ b/server/graphman/src/entities/config_check.rs @@ -0,0 +1,21 @@ +use async_graphql::SimpleObject; + +#[derive(Clone, Debug, SimpleObject)] +pub struct ConfigCheckResponse { + /// Checks if the config file is validated. + pub config_validated: bool, + /// Checks if the subgraph settings config set by GRAPH_EXPERIMENTAL_SUBGRAPH_SETTINGS are validated. + pub subgraph_settings_validated: bool, + /// Returns the Config file as a string. + pub config: String, +} + +impl ConfigCheckResponse { + pub fn from(config_validated: bool, subgraph_settings_validated: bool, config: String) -> Self { + Self { + config_validated, + subgraph_settings_validated, + config, + } + } +} diff --git a/server/graphman/src/entities/mod.rs b/server/graphman/src/entities/mod.rs index 8f4b2d8c018..57f01aa8fbc 100644 --- a/server/graphman/src/entities/mod.rs +++ b/server/graphman/src/entities/mod.rs @@ -2,6 +2,7 @@ mod block_hash; mod block_number; mod block_ptr; mod command_kind; +mod config_check; mod deployment_info; mod deployment_selector; mod deployment_status; @@ -15,6 +16,7 @@ pub use self::block_hash::BlockHash; pub use self::block_number::BlockNumber; pub use self::block_ptr::BlockPtr; pub use self::command_kind::CommandKind; +pub use self::config_check::ConfigCheckResponse; pub use self::deployment_info::DeploymentInfo; pub use self::deployment_selector::DeploymentSelector; pub use self::deployment_status::DeploymentStatus; diff --git a/server/graphman/src/resolvers/config_query.rs b/server/graphman/src/resolvers/config_query.rs new file mode 100644 index 00000000000..3c76a2678de --- /dev/null +++ b/server/graphman/src/resolvers/config_query.rs @@ -0,0 +1,16 @@ +use async_graphql::Context; +use async_graphql::Object; +use async_graphql::Result; + +use crate::entities::ConfigCheckResponse; + +mod check; +pub struct ConfigQuery; + +#[Object] +impl ConfigQuery { + /// Check and validate the configuration file + pub async fn check(&self, ctx: &Context<'_>) -> Result { + check::run(ctx) + } +} diff --git a/server/graphman/src/resolvers/config_query/check.rs b/server/graphman/src/resolvers/config_query/check.rs new file mode 100644 index 00000000000..12d6d1f4ba5 --- /dev/null +++ b/server/graphman/src/resolvers/config_query/check.rs @@ -0,0 +1,14 @@ +use async_graphql::{Context, Result}; +use graphman::commands::config::check::check; + +use crate::{entities::ConfigCheckResponse, resolvers::context::GraphmanContext}; + +pub fn run(ctx: &Context<'_>) -> Result { + let ctx = GraphmanContext::new(ctx)?; + let res = check(&ctx.config, true)?; + Ok(ConfigCheckResponse::from( + res.validated, + res.validated_subgraph_settings, + res.config_json.unwrap_or_default(), + )) +} diff --git a/server/graphman/src/resolvers/context.rs b/server/graphman/src/resolvers/context.rs index 8cc3e819c6d..7c492479094 100644 --- a/server/graphman/src/resolvers/context.rs +++ b/server/graphman/src/resolvers/context.rs @@ -2,14 +2,20 @@ use std::sync::Arc; use async_graphql::Context; use async_graphql::Result; +use clap::Parser; +use graph::log::logger; +use graph::prelude::error; use graph_store_postgres::connection_pool::ConnectionPool; use graph_store_postgres::NotificationSender; use graph_store_postgres::Store; +use graphman::config::Config; +use graphman::opt::Opt; pub struct GraphmanContext { pub primary_pool: ConnectionPool, pub notification_sender: Arc, pub store: Arc, + pub config: Config, } impl GraphmanContext { @@ -17,11 +23,21 @@ impl GraphmanContext { let primary_pool = ctx.data::()?.to_owned(); let notification_sender = ctx.data::>()?.to_owned(); let store = ctx.data::>()?.to_owned(); + let opt = Opt::parse(); + let logger = logger(opt.debug); + let config = match Config::load(&logger, &opt.clone().into()) { + Ok(config) => config, + Err(e) => { + error!(logger, "Failed to load config due to: {}", e.to_string()); + return Err(e.into()); + } + }; Ok(GraphmanContext { primary_pool, notification_sender, store, + config, }) } } diff --git a/server/graphman/src/resolvers/mod.rs b/server/graphman/src/resolvers/mod.rs index 2f7f225f6f4..f10854bd87b 100644 --- a/server/graphman/src/resolvers/mod.rs +++ b/server/graphman/src/resolvers/mod.rs @@ -1,3 +1,4 @@ +mod config_query; mod context; mod deployment_mutation; mod deployment_query; @@ -5,6 +6,7 @@ mod execution_query; mod mutation_root; mod query_root; +pub use self::config_query::ConfigQuery; pub use self::deployment_mutation::DeploymentMutation; pub use self::deployment_query::DeploymentQuery; pub use self::execution_query::ExecutionQuery; diff --git a/server/graphman/src/resolvers/query_root.rs b/server/graphman/src/resolvers/query_root.rs index 1c105abe40a..8ad3818280e 100644 --- a/server/graphman/src/resolvers/query_root.rs +++ b/server/graphman/src/resolvers/query_root.rs @@ -1,5 +1,6 @@ use async_graphql::Object; +use crate::resolvers::ConfigQuery; use crate::resolvers::DeploymentQuery; use crate::resolvers::ExecutionQuery; @@ -13,6 +14,11 @@ impl QueryRoot { DeploymentQuery {} } + /// Queries related to Config. + pub async fn config(&self) -> ConfigQuery { + ConfigQuery {} + } + /// Queries related to command executions. pub async fn execution(&self) -> ExecutionQuery { ExecutionQuery {} From f6018be3b474b2d5f2e3cba75e35931b10cc46c3 Mon Sep 17 00:00:00 2001 From: shiyasmohd Date: Wed, 1 Jan 2025 15:43:53 +0530 Subject: [PATCH 3/9] core: migrate test config to graphman(core) --- {node => core/graphman}/resources/tests/full_config.toml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {node => core/graphman}/resources/tests/full_config.toml (100%) diff --git a/node/resources/tests/full_config.toml b/core/graphman/resources/tests/full_config.toml similarity index 100% rename from node/resources/tests/full_config.toml rename to core/graphman/resources/tests/full_config.toml From 4e7824e93d72b19d7927ea31a3a692ffce18d45d Mon Sep 17 00:00:00 2001 From: shiyasmohd Date: Sat, 4 Jan 2025 10:47:20 +0530 Subject: [PATCH 4/9] server: accept only arguments in Opt when loading config in graphql api --- server/graphman/src/resolvers/config_query.rs | 5 +- .../src/resolvers/config_query/check.rs | 66 +++++++++++++++++-- server/graphman/src/resolvers/context.rs | 17 ----- 3 files changed, 62 insertions(+), 26 deletions(-) diff --git a/server/graphman/src/resolvers/config_query.rs b/server/graphman/src/resolvers/config_query.rs index 3c76a2678de..79c5ce8fa24 100644 --- a/server/graphman/src/resolvers/config_query.rs +++ b/server/graphman/src/resolvers/config_query.rs @@ -1,4 +1,3 @@ -use async_graphql::Context; use async_graphql::Object; use async_graphql::Result; @@ -10,7 +9,7 @@ pub struct ConfigQuery; #[Object] impl ConfigQuery { /// Check and validate the configuration file - pub async fn check(&self, ctx: &Context<'_>) -> Result { - check::run(ctx) + pub async fn check(&self) -> Result { + check::run() } } diff --git a/server/graphman/src/resolvers/config_query/check.rs b/server/graphman/src/resolvers/config_query/check.rs index 12d6d1f4ba5..ae9d35f6040 100644 --- a/server/graphman/src/resolvers/config_query/check.rs +++ b/server/graphman/src/resolvers/config_query/check.rs @@ -1,14 +1,68 @@ -use async_graphql::{Context, Result}; -use graphman::commands::config::check::check; +use async_graphql::Result; +use clap::Parser; +use graph::{log::logger, prelude::error}; +use graphman::{commands::config::check::check, config::Config, opt::Opt}; -use crate::{entities::ConfigCheckResponse, resolvers::context::GraphmanContext}; +use crate::entities::ConfigCheckResponse; + +pub fn run() -> Result { + let config = fetch_config()?; + let res = check(&config, true)?; -pub fn run(ctx: &Context<'_>) -> Result { - let ctx = GraphmanContext::new(ctx)?; - let res = check(&ctx.config, true)?; Ok(ConfigCheckResponse::from( res.validated, res.validated_subgraph_settings, res.config_json.unwrap_or_default(), )) } + +pub fn fetch_config() -> Result { + let args: Vec = std::env::args().collect(); + let accepted_flags = vec![ + "--config", + "--check-config", + "--subgraph", + "--start-block", + "--postgres-url", + "--postgres-secondary-hosts", + "--postgres-host-weights", + "--ethereum-rpc", + "--ethereum-ws", + "--ethereum-ipc", + "--ipfs", + "--arweave", + "--http-port", + "--index-node-port", + "--ws-port", + "--admin-port", + "--metrics-port", + "--node-id", + "--expensive-queries-filename", + "--debug", + "--elasticsearch-url", + "--elasticsearch-user", + "--elasticsearch-password", + "--disable-block-ingestor", + "--store-connection-pool-size", + "--unsafe-config", + "--debug-fork", + "--fork-base", + "--graphman-port", + ]; + let mut filtered_args: Vec = vec![args[0].clone()]; + for (i, arg) in args.iter().enumerate() { + if accepted_flags.contains(&arg.as_str()) { + filtered_args.push(arg.clone()); + filtered_args.push(args[i + 1].clone()); + } + } + let opt = Opt::try_parse_from(filtered_args).expect("Failed to parse args"); + let logger = logger(opt.debug); + match Config::load(&logger, &opt.clone().into()) { + Ok(config) => Ok(config), + Err(e) => { + error!(logger, "Failed to load config due to: {}", e.to_string()); + return Err(e.into()); + } + } +} diff --git a/server/graphman/src/resolvers/context.rs b/server/graphman/src/resolvers/context.rs index 7c492479094..e3e29ebef70 100644 --- a/server/graphman/src/resolvers/context.rs +++ b/server/graphman/src/resolvers/context.rs @@ -2,20 +2,14 @@ use std::sync::Arc; use async_graphql::Context; use async_graphql::Result; -use clap::Parser; -use graph::log::logger; -use graph::prelude::error; use graph_store_postgres::connection_pool::ConnectionPool; use graph_store_postgres::NotificationSender; use graph_store_postgres::Store; -use graphman::config::Config; -use graphman::opt::Opt; pub struct GraphmanContext { pub primary_pool: ConnectionPool, pub notification_sender: Arc, pub store: Arc, - pub config: Config, } impl GraphmanContext { @@ -23,21 +17,10 @@ impl GraphmanContext { let primary_pool = ctx.data::()?.to_owned(); let notification_sender = ctx.data::>()?.to_owned(); let store = ctx.data::>()?.to_owned(); - let opt = Opt::parse(); - let logger = logger(opt.debug); - let config = match Config::load(&logger, &opt.clone().into()) { - Ok(config) => config, - Err(e) => { - error!(logger, "Failed to load config due to: {}", e.to_string()); - return Err(e.into()); - } - }; - Ok(GraphmanContext { primary_pool, notification_sender, store, - config, }) } } From cd629221cb0cdf8c7c6237fdcac04037e3338361 Mon Sep 17 00:00:00 2001 From: shiyasmohd Date: Sat, 4 Jan 2025 16:23:01 +0530 Subject: [PATCH 5/9] server: add tests for graphman config check graphql api --- server/graphman/tests/config_query.rs | 101 ++++++++++++++++++ server/graphman/tests/test_config.toml | 71 ++++++++++++ .../tests/test_subgraph_settings.toml | 11 ++ 3 files changed, 183 insertions(+) create mode 100644 server/graphman/tests/config_query.rs create mode 100644 server/graphman/tests/test_config.toml create mode 100644 server/graphman/tests/test_subgraph_settings.toml diff --git a/server/graphman/tests/config_query.rs b/server/graphman/tests/config_query.rs new file mode 100644 index 00000000000..e1caaba8feb --- /dev/null +++ b/server/graphman/tests/config_query.rs @@ -0,0 +1,101 @@ +pub mod util; + +use graph::log::logger; +use graphman::config::Config; +use serde_json::json; + +use self::util::client::send_graphql_request; +use self::util::run_test; +use self::util::server::VALID_TOKEN; + +#[test] +fn graphql_can_validate_config_and_subgraphs_settings_config() { + run_test(|| async { + let curr_dir = std::env::current_dir() + .unwrap() + .to_str() + .unwrap() + .to_string(); + let config_dir = format!("{}/tests/test_config.toml", curr_dir); + let subgraph_settings_dir = format!("{}/tests/test_subgraph_settings.toml", curr_dir); + std::env::set_var("GRAPH_NODE_CONFIG", config_dir); + std::env::set_var( + "GRAPH_EXPERIMENTAL_SUBGRAPH_SETTINGS", + subgraph_settings_dir, + ); + + let resp = send_graphql_request( + json!({ + "query": r#"{ + config { + check { + configValidated + subgraphSettingsValidated + } + } + }"# + }), + VALID_TOKEN, + ) + .await; + + let expected_resp = json!({ + "data": { + "config": { + "check": { + "configValidated": true, + "subgraphSettingsValidated": true + } + } + } + }); + + assert_eq!(resp, expected_resp); + }); +} + +#[test] +fn graphql_can_return_config_as_json_string() { + run_test(|| async { + let curr_dir = std::env::current_dir() + .unwrap() + .to_str() + .unwrap() + .to_string(); + let config_dir = format!("{}/tests/test_config.toml", curr_dir); + std::env::set_var("GRAPH_NODE_CONFIG", config_dir.clone()); + + let resp = send_graphql_request( + json!({ + "query": r#"{ + config { + check { + config + } + } + }"# + }), + VALID_TOKEN, + ) + .await; + + let config = Config::from_file(&logger(true), &config_dir, "default") + .unwrap() + .to_json() + .unwrap(); + + assert_eq!( + resp.get("data") + .unwrap() + .get("config") + .unwrap() + .get("check") + .unwrap() + .get("config") + .unwrap() + .as_str() + .unwrap(), + config + ); + }); +} diff --git a/server/graphman/tests/test_config.toml b/server/graphman/tests/test_config.toml new file mode 100644 index 00000000000..1f907539194 --- /dev/null +++ b/server/graphman/tests/test_config.toml @@ -0,0 +1,71 @@ +[general] +query = "query_node_.*" + +[store] +[store.primary] +connection = "postgresql://postgres:1.1.1.1@test/primary" +pool_size = [ + { node = "index_node_1_.*", size = 2 }, + { node = "index_node_2_.*", size = 10 }, + { node = "index_node_3_.*", size = 10 }, + { node = "index_node_4_.*", size = 2 }, + { node = "query_node_.*", size = 10 } +] + +[store.shard_a] +connection = "postgresql://postgres:1.1.1.1@test/shard-a" +pool_size = [ + { node = "index_node_1_.*", size = 2 }, + { node = "index_node_2_.*", size = 10 }, + { node = "index_node_3_.*", size = 10 }, + { node = "index_node_4_.*", size = 2 }, + { node = "query_node_.*", size = 10 } +] + +[deployment] +# Studio subgraphs +[[deployment.rule]] +match = { name = "^prefix/" } +shard = "shard_a" +indexers = [ "index_prefix_0", + "index_prefix_1" ] + +[[deployment.rule]] +match = { name = "^custom/.*" } +indexers = [ "index_custom_0" ] + +[[deployment.rule]] +shards = [ "primary", "shard_a" ] +indexers = [ "index_node_1_a", + "index_node_2_a", + "index_node_3_a" ] + +[chains] +ingestor = "index_0" + +[chains.mainnet] +shard = "primary" +provider = [ + { label = "mainnet-0", url = "http://rpc.mainnet.io", features = ["archive", "traces"] }, + { label = "mainnet-1", details = { type = "web3call", url = "http://rpc.mainnet.io", features = ["archive", "traces"] }}, + { label = "firehose", details = { type = "firehose", url = "http://localhost:9000", features = [] }}, + { label = "substreams", details = { type = "substreams", url = "http://localhost:9000", features = [] }}, +] + +[chains.ropsten] +shard = "primary" +provider = [ + { label = "ropsten-0", url = "http://rpc.ropsten.io", transport = "rpc", features = ["archive", "traces"] } +] + +[chains.goerli] +shard = "primary" +provider = [ + { label = "goerli-0", url = "http://rpc.goerli.io", transport = "ipc", features = ["archive"] } +] + +[chains.kovan] +shard = "primary" +provider = [ + { label = "kovan-0", url = "http://rpc.kovan.io", transport = "ws", features = [] } +] diff --git a/server/graphman/tests/test_subgraph_settings.toml b/server/graphman/tests/test_subgraph_settings.toml new file mode 100644 index 00000000000..86a500c037b --- /dev/null +++ b/server/graphman/tests/test_subgraph_settings.toml @@ -0,0 +1,11 @@ +[[setting]] +match = { name = ".*" } +history_blocks = 10000 + +[[setting]] +match = { name = "xxxxx" } +history_blocks = 10000 + +[[setting]] +match = { name = ".*!$" } +history_blocks = 10000 From dcd0ca9ff5bce097a15f1c26d33f28465f88fd0e Mon Sep 17 00:00:00 2001 From: shiyasmohd Date: Sun, 5 Jan 2025 19:22:36 +0530 Subject: [PATCH 6/9] core,node,server: add graphman config place command to graphql api --- core/graphman/src/commands/config/mod.rs | 1 + core/graphman/src/commands/config/place.rs | 32 +++++++++++++++++++ node/src/bin/manager.rs | 2 +- node/src/manager/commands/config_cmd/mod.rs | 1 + node/src/manager/commands/config_cmd/place.rs | 22 +++++++++++++ server/graphman/src/entities/mod.rs | 2 ++ .../graphman/src/entities/place_response.rs | 25 +++++++++++++++ server/graphman/src/resolvers/config_query.rs | 11 +++++++ .../src/resolvers/config_query/place.rs | 15 +++++++++ 9 files changed, 110 insertions(+), 1 deletion(-) create mode 100644 core/graphman/src/commands/config/place.rs create mode 100644 node/src/manager/commands/config_cmd/place.rs create mode 100644 server/graphman/src/entities/place_response.rs create mode 100644 server/graphman/src/resolvers/config_query/place.rs diff --git a/core/graphman/src/commands/config/mod.rs b/core/graphman/src/commands/config/mod.rs index 3e8ff0fc6bd..866c78cff79 100644 --- a/core/graphman/src/commands/config/mod.rs +++ b/core/graphman/src/commands/config/mod.rs @@ -1 +1,2 @@ pub mod check; +pub mod place; diff --git a/core/graphman/src/commands/config/place.rs b/core/graphman/src/commands/config/place.rs new file mode 100644 index 00000000000..2b6f88ca3fb --- /dev/null +++ b/core/graphman/src/commands/config/place.rs @@ -0,0 +1,32 @@ +use anyhow::{anyhow, Error as AnyError}; +use graph_store_postgres::DeploymentPlacer; +use thiserror::Error; + +pub struct PlaceResult { + pub shards: Vec, + pub nodes: Vec, +} + +#[derive(Debug, Error)] +pub enum PlaceError { + #[error("No matching placement rule; default placement from JSON RPC call would be used")] + NoPlacementRule, + + #[error(transparent)] + Common(#[from] AnyError), +} + +pub fn place( + placer: &dyn DeploymentPlacer, + name: &str, + network: &str, +) -> Result { + match placer.place(name, network).map_err(|s| anyhow!(s))? { + None => Err(PlaceError::NoPlacementRule), + Some((shards, nodes)) => { + let nodes: Vec<_> = nodes.into_iter().map(|n| n.to_string()).collect(); + let shards: Vec<_> = shards.into_iter().map(|s| s.to_string()).collect(); + Ok(PlaceResult { shards, nodes }) + } + } +} diff --git a/node/src/bin/manager.rs b/node/src/bin/manager.rs index a466c72f364..f0484deea4b 100644 --- a/node/src/bin/manager.rs +++ b/node/src/bin/manager.rs @@ -1182,7 +1182,7 @@ async fn main() -> anyhow::Result<()> { Ok(()) } Place { name, network } => { - commands::config::place(&ctx.config.deployment, &name, &network) + commands::config_cmd::place::run(&ctx.config.deployment, &name, &network) } Check { print } => commands::config_cmd::check::run(&ctx.config, print), Pools { nodes, shard } => commands::config::pools(&ctx.config, nodes, shard), diff --git a/node/src/manager/commands/config_cmd/mod.rs b/node/src/manager/commands/config_cmd/mod.rs index 3e8ff0fc6bd..866c78cff79 100644 --- a/node/src/manager/commands/config_cmd/mod.rs +++ b/node/src/manager/commands/config_cmd/mod.rs @@ -1 +1,2 @@ pub mod check; +pub mod place; diff --git a/node/src/manager/commands/config_cmd/place.rs b/node/src/manager/commands/config_cmd/place.rs new file mode 100644 index 00000000000..172d4b68257 --- /dev/null +++ b/node/src/manager/commands/config_cmd/place.rs @@ -0,0 +1,22 @@ +use anyhow::Error; +use graph_store_postgres::DeploymentPlacer; +use graphman::commands::config::place::{place, PlaceError}; + +pub fn run(placer: &dyn DeploymentPlacer, name: &String, network: &String) -> Result<(), Error> { + let res = place(placer, name, network); + match res { + Ok(result) => { + println!("subgraph: {}", name); + println!("network: {}", network); + println!("shard: {}", result.shards.join(", ")); + println!("nodes: {}", result.nodes.join(", ")); + } + Err(PlaceError::NoPlacementRule) => { + println!("{}", PlaceError::NoPlacementRule); + } + Err(PlaceError::Common(e)) => { + println!("Error: {}", e.to_string()); + } + }; + Ok(()) +} diff --git a/server/graphman/src/entities/mod.rs b/server/graphman/src/entities/mod.rs index 57f01aa8fbc..58a00156ab9 100644 --- a/server/graphman/src/entities/mod.rs +++ b/server/graphman/src/entities/mod.rs @@ -10,6 +10,7 @@ mod deployment_version_selector; mod empty_response; mod execution; mod execution_id; +mod place_response; mod subgraph_health; pub use self::block_hash::BlockHash; @@ -24,4 +25,5 @@ pub use self::deployment_version_selector::DeploymentVersionSelector; pub use self::empty_response::EmptyResponse; pub use self::execution::Execution; pub use self::execution_id::ExecutionId; +pub use self::place_response::PlaceResponse; pub use self::subgraph_health::SubgraphHealth; diff --git a/server/graphman/src/entities/place_response.rs b/server/graphman/src/entities/place_response.rs new file mode 100644 index 00000000000..83d05db5545 --- /dev/null +++ b/server/graphman/src/entities/place_response.rs @@ -0,0 +1,25 @@ +use async_graphql::SimpleObject; + +#[derive(Clone, Debug, SimpleObject)] +pub struct PlaceResponse { + subgraph: String, + network: String, + shards: Vec, + nodes: Vec, +} + +impl PlaceResponse { + pub fn from( + subgraph: String, + network: String, + shards: Vec, + nodes: Vec, + ) -> Self { + Self { + subgraph, + network, + shards, + nodes, + } + } +} diff --git a/server/graphman/src/resolvers/config_query.rs b/server/graphman/src/resolvers/config_query.rs index 79c5ce8fa24..370ccf45933 100644 --- a/server/graphman/src/resolvers/config_query.rs +++ b/server/graphman/src/resolvers/config_query.rs @@ -2,8 +2,10 @@ use async_graphql::Object; use async_graphql::Result; use crate::entities::ConfigCheckResponse; +use crate::entities::PlaceResponse; mod check; +mod place; pub struct ConfigQuery; #[Object] @@ -12,4 +14,13 @@ impl ConfigQuery { pub async fn check(&self) -> Result { check::run() } + + /// Returns how a specific subgraph would be placed + pub async fn place( + &self, + #[graphql(desc = "Subgraph name")] subgraph: String, + #[graphql(desc = "Network name")] network: String, + ) -> Result { + place::run(subgraph, network).await + } } diff --git a/server/graphman/src/resolvers/config_query/place.rs b/server/graphman/src/resolvers/config_query/place.rs new file mode 100644 index 00000000000..16ea2507ea4 --- /dev/null +++ b/server/graphman/src/resolvers/config_query/place.rs @@ -0,0 +1,15 @@ +use async_graphql::Result; +use graphman::commands::config::place::place; + +use crate::entities::PlaceResponse; + +use super::check::fetch_config; + +pub async fn run(subgraph: String, network: String) -> Result { + let config = fetch_config()?; + let res = place(&config.deployment, &subgraph, &network)?; + + Ok(PlaceResponse::from( + subgraph, network, res.shards, res.nodes, + )) +} From 03ae4e8477fd5d86fecc8c63bb6f2733db40f045 Mon Sep 17 00:00:00 2001 From: shiyasmohd Date: Tue, 7 Jan 2025 07:10:57 +0530 Subject: [PATCH 7/9] server: add test for graphman config place graphql api --- server/graphman/tests/config_query.rs | 76 +++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/server/graphman/tests/config_query.rs b/server/graphman/tests/config_query.rs index e1caaba8feb..0f41065c597 100644 --- a/server/graphman/tests/config_query.rs +++ b/server/graphman/tests/config_query.rs @@ -99,3 +99,79 @@ fn graphql_can_return_config_as_json_string() { ); }); } + +#[test] +fn graphql_can_check_how_specific_subgraph_would_be_placed() { + run_test(|| async { + let curr_dir = std::env::current_dir() + .unwrap() + .to_str() + .unwrap() + .to_string(); + let config_dir = format!("{}/tests/test_config.toml", curr_dir); + std::env::set_var("GRAPH_NODE_CONFIG", config_dir); + + let resp = send_graphql_request( + json!({ + "query": r#"{ + config { + place(subgraph: "subgraph_1", network: "bsc") { + subgraph + network + shards + nodes + } + } + }"# + }), + VALID_TOKEN, + ) + .await; + + let expected_resp = json!({ + "data": { + "config": { + "place": { + "subgraph": String::from("subgraph_1"), + "network": String::from("bsc"), + "shards": vec!["primary".to_string(),"shard_a".to_string()], + "nodes": vec!["index_node_1_a".to_string(),"index_node_2_a".to_string(),"index_node_3_a".to_string()], + } + } + } + }); + + let resp_custom = send_graphql_request( + json!({ + "query": r#"{ + config { + place(subgraph: "custom/subgraph_1", network: "bsc") { + subgraph + network + shards + nodes + } + } + }"# + }), + VALID_TOKEN, + ) + .await; + + let expected_resp_custom = json!({ + "data": { + "config": { + "place": { + "subgraph": String::from("custom/subgraph_1"), + "network": String::from("bsc"), + "shards": vec!["primary".to_string()], + "nodes": vec!["index_custom_0".to_string()], + } + } + } + }); + + assert_eq!(resp, expected_resp); + assert_eq!(resp_custom, expected_resp_custom); + }); +} From 91432b98c4311b970629e7a3fc6e1fa76e6cc297 Mon Sep 17 00:00:00 2001 From: shiyasmohd Date: Wed, 8 Jan 2025 07:26:33 +0530 Subject: [PATCH 8/9] core,node,server: add graphman config pools to graphql api --- core/graphman/src/commands/config/mod.rs | 1 + core/graphman/src/commands/config/pools.rs | 81 +++++++++++++++++++ node/src/bin/manager.rs | 4 +- node/src/manager/commands/config_cmd/mod.rs | 1 + node/src/manager/commands/config_cmd/pools.rs | 22 +++++ server/graphman/src/entities/mod.rs | 2 + server/graphman/src/entities/pools.rs | 12 +++ server/graphman/src/resolvers/config_query.rs | 10 +++ .../src/resolvers/config_query/pools.rs | 19 +++++ 9 files changed, 151 insertions(+), 1 deletion(-) create mode 100644 core/graphman/src/commands/config/pools.rs create mode 100644 node/src/manager/commands/config_cmd/pools.rs create mode 100644 server/graphman/src/entities/pools.rs create mode 100644 server/graphman/src/resolvers/config_query/pools.rs diff --git a/core/graphman/src/commands/config/mod.rs b/core/graphman/src/commands/config/mod.rs index 866c78cff79..db0b34e44da 100644 --- a/core/graphman/src/commands/config/mod.rs +++ b/core/graphman/src/commands/config/mod.rs @@ -1,2 +1,3 @@ pub mod check; pub mod place; +pub mod pools; diff --git a/core/graphman/src/commands/config/pools.rs b/core/graphman/src/commands/config/pools.rs new file mode 100644 index 00000000000..ec03b3b9e8e --- /dev/null +++ b/core/graphman/src/commands/config/pools.rs @@ -0,0 +1,81 @@ +use std::collections::BTreeMap; + +use anyhow::Error as AnyError; +use async_graphql::SimpleObject; +use graph::prelude::NodeId; +use thiserror::Error; + +use crate::config::Config; + +#[derive(Debug, Error)] +pub enum PoolsError { + #[error("illegal node name `{0}`")] + IllegalNodeName(String), + #[error(transparent)] + Common(#[from] AnyError), +} + +#[derive(Clone, Debug, SimpleObject)] +pub struct Pools { + pub shards: BTreeMap, + pub node: String, +} + +#[derive(Debug)] +pub struct PoolsResult { + pub pools: Vec, + pub shards: BTreeMap, +} + +pub fn pools(config: &Config, nodes: &Vec) -> Result { + // Quietly replace `-` with `_` in node names to make passing in pod names + // from k8s less annoying + let nodes: Vec<_> = nodes + .into_iter() + .map(|name| { + NodeId::new(name.replace('-', "_")) + .map_err(|()| PoolsError::IllegalNodeName(name.to_string())) + }) + .collect::>()?; + // node -> shard_name -> size + let mut sizes = BTreeMap::new(); + for node in &nodes { + let mut shard_sizes = BTreeMap::new(); + for (name, shard) in &config.stores { + let size = shard + .pool_size + .size_for(node, name) + .map_err(PoolsError::Common)?; + shard_sizes.insert(name.to_string(), size); + for (replica_name, replica) in &shard.replicas { + let qname = format!("{}.{}", name, replica_name); + let size = replica + .pool_size + .size_for(node, &qname) + .map_err(PoolsError::Common)?; + shard_sizes.insert(qname, size); + } + } + sizes.insert(node.to_string(), shard_sizes); + } + + let mut by_shard: BTreeMap = BTreeMap::new(); + for shard_sizes in sizes.values() { + for (shard_name, size) in shard_sizes { + *by_shard.entry(shard_name.to_string()).or_default() += size; + } + } + let mut pools: Vec = Vec::new(); + for node in &nodes { + let empty = BTreeMap::new(); + let node_sizes = sizes.get(node.as_str()).unwrap_or(&empty); + pools.push(Pools { + shards: node_sizes.clone(), + node: node.to_string(), + }) + } + Ok(PoolsResult { + pools, + shards: by_shard, + }) +} diff --git a/node/src/bin/manager.rs b/node/src/bin/manager.rs index f0484deea4b..7432b0b69c4 100644 --- a/node/src/bin/manager.rs +++ b/node/src/bin/manager.rs @@ -1185,7 +1185,9 @@ async fn main() -> anyhow::Result<()> { commands::config_cmd::place::run(&ctx.config.deployment, &name, &network) } Check { print } => commands::config_cmd::check::run(&ctx.config, print), - Pools { nodes, shard } => commands::config::pools(&ctx.config, nodes, shard), + Pools { nodes, shard } => { + commands::config_cmd::pools::run(&ctx.config, nodes, shard) + } Provider { features, network } => { let logger = ctx.logger.clone(); let registry = ctx.registry.clone(); diff --git a/node/src/manager/commands/config_cmd/mod.rs b/node/src/manager/commands/config_cmd/mod.rs index 866c78cff79..db0b34e44da 100644 --- a/node/src/manager/commands/config_cmd/mod.rs +++ b/node/src/manager/commands/config_cmd/mod.rs @@ -1,2 +1,3 @@ pub mod check; pub mod place; +pub mod pools; diff --git a/node/src/manager/commands/config_cmd/pools.rs b/node/src/manager/commands/config_cmd/pools.rs new file mode 100644 index 00000000000..d322e1ceaad --- /dev/null +++ b/node/src/manager/commands/config_cmd/pools.rs @@ -0,0 +1,22 @@ +use anyhow::Error; +use graphman::{commands::config::pools::pools, config::Config}; + +pub fn run(config: &Config, nodes: Vec, shard: bool) -> Result<(), Error> { + if nodes.is_empty() { + return Err(Error::msg("No nodes specified")); + } + let res = pools(config, &nodes)?; + if shard { + for shard in res.shards { + println!("{}: {}", shard.0, shard.1); + } + } else { + for pool in res.pools { + println!("{}:", pool.node); + for shard in pool.shards { + println!(" {}: {}", shard.0, shard.1); + } + } + } + Ok(()) +} diff --git a/server/graphman/src/entities/mod.rs b/server/graphman/src/entities/mod.rs index 58a00156ab9..2ff6a0e4e19 100644 --- a/server/graphman/src/entities/mod.rs +++ b/server/graphman/src/entities/mod.rs @@ -11,6 +11,7 @@ mod empty_response; mod execution; mod execution_id; mod place_response; +mod pools; mod subgraph_health; pub use self::block_hash::BlockHash; @@ -26,4 +27,5 @@ pub use self::empty_response::EmptyResponse; pub use self::execution::Execution; pub use self::execution_id::ExecutionId; pub use self::place_response::PlaceResponse; +pub use self::pools::PoolsResponse; pub use self::subgraph_health::SubgraphHealth; diff --git a/server/graphman/src/entities/pools.rs b/server/graphman/src/entities/pools.rs new file mode 100644 index 00000000000..ad394080ad2 --- /dev/null +++ b/server/graphman/src/entities/pools.rs @@ -0,0 +1,12 @@ +use std::collections::BTreeMap; + +use async_graphql::SimpleObject; +use graphman::commands::config::pools::Pools; + +#[derive(Clone, Debug, SimpleObject)] +pub struct PoolsResponse { + /// Size of database pools for each node. + pub pools: Vec, + /// Connections by shard rather than by node + pub shards: BTreeMap, +} diff --git a/server/graphman/src/resolvers/config_query.rs b/server/graphman/src/resolvers/config_query.rs index 370ccf45933..3ae316650fb 100644 --- a/server/graphman/src/resolvers/config_query.rs +++ b/server/graphman/src/resolvers/config_query.rs @@ -3,9 +3,11 @@ use async_graphql::Result; use crate::entities::ConfigCheckResponse; use crate::entities::PlaceResponse; +use crate::entities::PoolsResponse; mod check; mod place; +mod pools; pub struct ConfigQuery; #[Object] @@ -23,4 +25,12 @@ impl ConfigQuery { ) -> Result { place::run(subgraph, network).await } + + // Information about the size of database pools + pub async fn pools( + &self, + #[graphql(desc = "The names of the nodes that are going to run")] nodes: Vec, + ) -> Result { + pools::run(nodes).await + } } diff --git a/server/graphman/src/resolvers/config_query/pools.rs b/server/graphman/src/resolvers/config_query/pools.rs new file mode 100644 index 00000000000..5e3f92a0734 --- /dev/null +++ b/server/graphman/src/resolvers/config_query/pools.rs @@ -0,0 +1,19 @@ +use async_graphql::Result; +use graphman::commands::config::pools::pools; + +use crate::entities::PoolsResponse; + +use super::check::fetch_config; + +pub async fn run(nodes: Vec) -> Result { + let config = fetch_config()?; + let res = pools(&config, &nodes); + + match res { + Ok(res) => Ok(PoolsResponse { + pools: res.pools, + shards: res.shards, + }), + Err(e) => Err(e.into()), + } +} From 359df2b2a91619cb6b3d961f10b005ea061e2789 Mon Sep 17 00:00:00 2001 From: shiyasmohd Date: Wed, 8 Jan 2025 12:49:23 +0530 Subject: [PATCH 9/9] server: add tests for graphman config pools graphql api --- server/graphman/tests/config_query.rs | 106 ++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) diff --git a/server/graphman/tests/config_query.rs b/server/graphman/tests/config_query.rs index 0f41065c597..3e19111bb04 100644 --- a/server/graphman/tests/config_query.rs +++ b/server/graphman/tests/config_query.rs @@ -175,3 +175,109 @@ fn graphql_can_check_how_specific_subgraph_would_be_placed() { assert_eq!(resp_custom, expected_resp_custom); }); } + +#[test] +fn graphql_can_fetch_info_about_size_of_database_pools() { + run_test(|| async { + let curr_dir = std::env::current_dir() + .unwrap() + .to_str() + .unwrap() + .to_string(); + let config_dir = format!("{}/tests/test_config.toml", curr_dir); + std::env::set_var("GRAPH_NODE_CONFIG", config_dir); + + let resp = send_graphql_request( + json!({ + "query": r#"{ + config { + pools(nodes: ["index_node_1_a", "index_node_2_a", "index_node_3_a"]) { + pools { + shards + node + } + } + } + }"# + }), + VALID_TOKEN, + ) + .await; + + let expected_resp = json!({ + "data": { + "config": { + "pools": { + "pools": [ + { + "shards": { + "primary": 2, + "shard_a": 2 + }, + "node": "index_node_1_a" + }, + { + "shards": { + "primary": 10, + "shard_a": 10 + }, + "node": "index_node_2_a" + }, + { + "shards": { + "primary": 10, + "shard_a": 10 + }, + "node": "index_node_3_a" + } + ] + } + } + } + }); + + assert_eq!(resp, expected_resp); + }); +} + +#[test] +fn graphql_can_fetch_connections_by_shard() { + run_test(|| async { + let curr_dir = std::env::current_dir() + .unwrap() + .to_str() + .unwrap() + .to_string(); + let config_dir = format!("{}/tests/test_config.toml", curr_dir); + std::env::set_var("GRAPH_NODE_CONFIG", config_dir); + + let resp = send_graphql_request( + json!({ + "query": r#"{ + config { + pools(nodes: ["index_node_1_a", "index_node_2_a", "index_node_3_a"]) { + shards + } + } + }"# + }), + VALID_TOKEN, + ) + .await; + + let expected_resp = json!({ + "data": { + "config": { + "pools": { + "shards": { + "primary": 22, + "shard_a": 22 + } + } + } + } + }); + + assert_eq!(resp, expected_resp); + }); +}