diff --git a/Cargo.lock b/Cargo.lock index 17573acd02c..3341661f11c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "Inflector" @@ -2183,6 +2183,7 @@ dependencies = [ "graph-runtime-wasm", "graph-server-index-node", "graph-store-postgres", + "graphman", "secp256k1", "serde", "serde_yaml", @@ -2207,13 +2208,21 @@ name = "graphman" version = "0.36.0" dependencies = [ "anyhow", + "async-graphql", + "clap", "diesel", + "git-testament", "graph", + "graph-chain-ethereum", "graph-store-postgres", "graphman-store", "itertools 0.13.0", + "lazy_static", + "serde", + "shellexpand", "thiserror", "tokio", + "url", ] [[package]] @@ -2225,6 +2234,7 @@ dependencies = [ "async-graphql-axum", "axum 0.7.5", "chrono", + "clap", "diesel", "graph", "graph-store-postgres", @@ -5034,6 +5044,7 @@ dependencies = [ "graph-graphql", "graph-node", "graph-store-postgres", + "graphman", "hex", "hex-literal 0.4.1", "lazy_static", diff --git a/core/graphman/Cargo.toml b/core/graphman/Cargo.toml index 001a683f4aa..c25d9ffadc2 100644 --- a/core/graphman/Cargo.toml +++ b/core/graphman/Cargo.toml @@ -5,10 +5,18 @@ edition.workspace = true [dependencies] anyhow = { workspace = true } +async-graphql = { workspace = true } +clap = { workspace = true } diesel = { workspace = true } +git-testament = "0.2" graph = { workspace = true } graph-store-postgres = { workspace = true } graphman-store = { workspace = true } itertools = { workspace = true } +lazy_static = "1.5.0" thiserror = { workspace = true } tokio = { workspace = true } +graph-chain-ethereum = { path = "../../chain/ethereum" } +serde = { workspace = true } +url = "2.5.2" +shellexpand = "3.1.0" diff --git a/node/resources/tests/full_config.toml b/core/graphman/resources/tests/full_config.toml similarity index 100% rename from node/resources/tests/full_config.toml rename to core/graphman/resources/tests/full_config.toml diff --git a/core/graphman/src/commands/config/check.rs b/core/graphman/src/commands/config/check.rs new file mode 100644 index 00000000000..5f9831b83f4 --- /dev/null +++ b/core/graphman/src/commands/config/check.rs @@ -0,0 +1,42 @@ +use crate::config::Config; +use anyhow::Error; +use graph::{components::subgraph::Settings, env::EnvVars}; + +pub struct ConfigCheckResult { + pub validated: bool, + pub validated_subgraph_settings: bool, + pub config_json: Option, +} + +pub fn check(config: &Config, print: bool) -> Result { + let mut result = ConfigCheckResult { + validated: false, + validated_subgraph_settings: false, + config_json: None, + }; + if print { + match config.to_json() { + Ok(txt) => { + result.config_json = Some(txt); + } + Err(err) => return Err(anyhow::format_err!("error serializing config: {}", err)), + }; + } + let env_vars = EnvVars::from_env().unwrap(); + if let Some(path) = &env_vars.subgraph_settings { + match Settings::from_file(path) { + Ok(_) => { + result.validated_subgraph_settings = true; + } + Err(e) => { + return Err(anyhow::format_err!( + "configuration error in subgraph settings {}: {}", + path, + e + )); + } + } + }; + result.validated = true; + Ok(result) +} diff --git a/core/graphman/src/commands/config/mod.rs b/core/graphman/src/commands/config/mod.rs new file mode 100644 index 00000000000..db0b34e44da --- /dev/null +++ b/core/graphman/src/commands/config/mod.rs @@ -0,0 +1,3 @@ +pub mod check; +pub mod place; +pub mod pools; diff --git a/core/graphman/src/commands/config/place.rs b/core/graphman/src/commands/config/place.rs new file mode 100644 index 00000000000..2b6f88ca3fb --- /dev/null +++ b/core/graphman/src/commands/config/place.rs @@ -0,0 +1,32 @@ +use anyhow::{anyhow, Error as AnyError}; +use graph_store_postgres::DeploymentPlacer; +use thiserror::Error; + +pub struct PlaceResult { + pub shards: Vec, + pub nodes: Vec, +} + +#[derive(Debug, Error)] +pub enum PlaceError { + #[error("No matching placement rule; default placement from JSON RPC call would be used")] + NoPlacementRule, + + #[error(transparent)] + Common(#[from] AnyError), +} + +pub fn place( + placer: &dyn DeploymentPlacer, + name: &str, + network: &str, +) -> Result { + match placer.place(name, network).map_err(|s| anyhow!(s))? { + None => Err(PlaceError::NoPlacementRule), + Some((shards, nodes)) => { + let nodes: Vec<_> = nodes.into_iter().map(|n| n.to_string()).collect(); + let shards: Vec<_> = shards.into_iter().map(|s| s.to_string()).collect(); + Ok(PlaceResult { shards, nodes }) + } + } +} diff --git a/core/graphman/src/commands/config/pools.rs b/core/graphman/src/commands/config/pools.rs new file mode 100644 index 00000000000..ec03b3b9e8e --- /dev/null +++ b/core/graphman/src/commands/config/pools.rs @@ -0,0 +1,81 @@ +use std::collections::BTreeMap; + +use anyhow::Error as AnyError; +use async_graphql::SimpleObject; +use graph::prelude::NodeId; +use thiserror::Error; + +use crate::config::Config; + +#[derive(Debug, Error)] +pub enum PoolsError { + #[error("illegal node name `{0}`")] + IllegalNodeName(String), + #[error(transparent)] + Common(#[from] AnyError), +} + +#[derive(Clone, Debug, SimpleObject)] +pub struct Pools { + pub shards: BTreeMap, + pub node: String, +} + +#[derive(Debug)] +pub struct PoolsResult { + pub pools: Vec, + pub shards: BTreeMap, +} + +pub fn pools(config: &Config, nodes: &Vec) -> Result { + // Quietly replace `-` with `_` in node names to make passing in pod names + // from k8s less annoying + let nodes: Vec<_> = nodes + .into_iter() + .map(|name| { + NodeId::new(name.replace('-', "_")) + .map_err(|()| PoolsError::IllegalNodeName(name.to_string())) + }) + .collect::>()?; + // node -> shard_name -> size + let mut sizes = BTreeMap::new(); + for node in &nodes { + let mut shard_sizes = BTreeMap::new(); + for (name, shard) in &config.stores { + let size = shard + .pool_size + .size_for(node, name) + .map_err(PoolsError::Common)?; + shard_sizes.insert(name.to_string(), size); + for (replica_name, replica) in &shard.replicas { + let qname = format!("{}.{}", name, replica_name); + let size = replica + .pool_size + .size_for(node, &qname) + .map_err(PoolsError::Common)?; + shard_sizes.insert(qname, size); + } + } + sizes.insert(node.to_string(), shard_sizes); + } + + let mut by_shard: BTreeMap = BTreeMap::new(); + for shard_sizes in sizes.values() { + for (shard_name, size) in shard_sizes { + *by_shard.entry(shard_name.to_string()).or_default() += size; + } + } + let mut pools: Vec = Vec::new(); + for node in &nodes { + let empty = BTreeMap::new(); + let node_sizes = sizes.get(node.as_str()).unwrap_or(&empty); + pools.push(Pools { + shards: node_sizes.clone(), + node: node.to_string(), + }) + } + Ok(PoolsResult { + pools, + shards: by_shard, + }) +} diff --git a/core/graphman/src/commands/mod.rs b/core/graphman/src/commands/mod.rs index 98629027b58..ffa528041a0 100644 --- a/core/graphman/src/commands/mod.rs +++ b/core/graphman/src/commands/mod.rs @@ -1 +1,2 @@ +pub mod config; pub mod deployment; diff --git a/node/src/config.rs b/core/graphman/src/config.rs similarity index 100% rename from node/src/config.rs rename to core/graphman/src/config.rs index 8006b8efef7..d486f69ae8f 100644 --- a/node/src/config.rs +++ b/core/graphman/src/config.rs @@ -19,9 +19,9 @@ use graph::{ use graph_chain_ethereum as ethereum; use graph_chain_ethereum::NodeCapabilities; use graph_store_postgres::{DeploymentPlacer, Shard as ShardName, PRIMARY_SHARD}; +use serde::Serialize; use graph::http::{HeaderMap, Uri}; -use serde::Serialize; use std::{ collections::{BTreeMap, BTreeSet}, fmt, diff --git a/core/graphman/src/lib.rs b/core/graphman/src/lib.rs index 71f8e77a848..d48bf9bbb44 100644 --- a/core/graphman/src/lib.rs +++ b/core/graphman/src/lib.rs @@ -8,8 +8,10 @@ mod error; pub mod commands; +pub mod config; pub mod deployment; pub mod execution_tracker; +pub mod opt; pub use self::error::GraphmanError; pub use self::execution_tracker::GraphmanExecutionTracker; diff --git a/node/src/opt.rs b/core/graphman/src/opt.rs similarity index 100% rename from node/src/opt.rs rename to core/graphman/src/opt.rs diff --git a/node/src/bin/manager.rs b/node/src/bin/manager.rs index afc09403357..7432b0b69c4 100644 --- a/node/src/bin/manager.rs +++ b/node/src/bin/manager.rs @@ -19,7 +19,6 @@ use graph::{ }; use graph_chain_ethereum::EthereumAdapter; use graph_graphql::prelude::GraphQlRunner; -use graph_node::config::{self, Config as Cfg}; use graph_node::manager::color::Terminal; use graph_node::manager::commands; use graph_node::network_setup::Networks; @@ -34,6 +33,7 @@ use graph_store_postgres::{ connection_pool::ConnectionPool, BlockStore, NotificationSender, Shard, Store, SubgraphStore, SubscriptionManager, PRIMARY_SHARD, }; +use graphman::config::{self, Config as Cfg}; use itertools::Itertools; use lazy_static::lazy_static; use std::str::FromStr; @@ -1182,10 +1182,12 @@ async fn main() -> anyhow::Result<()> { Ok(()) } Place { name, network } => { - commands::config::place(&ctx.config.deployment, &name, &network) + commands::config_cmd::place::run(&ctx.config.deployment, &name, &network) + } + Check { print } => commands::config_cmd::check::run(&ctx.config, print), + Pools { nodes, shard } => { + commands::config_cmd::pools::run(&ctx.config, nodes, shard) } - Check { print } => commands::config::check(&ctx.config, print), - Pools { nodes, shard } => commands::config::pools(&ctx.config, nodes, shard), Provider { features, network } => { let logger = ctx.logger.clone(); let registry = ctx.registry.clone(); diff --git a/node/src/chain.rs b/node/src/chain.rs index 1c62bf2248e..332b8367c87 100644 --- a/node/src/chain.rs +++ b/node/src/chain.rs @@ -1,4 +1,3 @@ -use crate::config::{Config, ProviderDetails}; use crate::network_setup::{ AdapterConfiguration, EthAdapterConfig, FirehoseAdapterConfig, Networks, }; @@ -31,6 +30,7 @@ use graph::tokio::time::timeout; use graph::url::Url; use graph_chain_ethereum::{self as ethereum, Transport}; use graph_store_postgres::{BlockStore, ChainHeadUpdateListener}; +use graphman::config::{Config, ProviderDetails}; use std::cmp::Ordering; use std::collections::BTreeMap; use std::sync::Arc; @@ -240,7 +240,7 @@ pub async fn create_ethereum_networks_for_chain( "capabilities" => capabilities ); - use crate::config::Transport::*; + use graphman::config::Transport::*; let transport = match web3.transport { Rpc => Transport::new_rpc( @@ -553,13 +553,13 @@ pub async fn networks_as_chains( #[cfg(test)] mod test { - use crate::config::{Config, Opt}; use crate::network_setup::{AdapterConfiguration, Networks}; use graph::components::network_provider::ChainName; use graph::endpoint::EndpointMetrics; use graph::log::logger; use graph::prelude::{tokio, MetricsRegistry}; use graph_chain_ethereum::NodeCapabilities; + use graphman::config::{Config, Opt}; use std::sync::Arc; #[tokio::test] diff --git a/node/src/lib.rs b/node/src/lib.rs index f65ffc1be8f..7f78083f1fa 100644 --- a/node/src/lib.rs +++ b/node/src/lib.rs @@ -6,9 +6,7 @@ use graph::{prelude::MetricsRegistry, prometheus::Registry}; extern crate diesel; pub mod chain; -pub mod config; pub mod network_setup; -pub mod opt; pub mod store_builder; pub mod manager; diff --git a/node/src/main.rs b/node/src/main.rs index 870cce97318..ada0f53876c 100644 --- a/node/src/main.rs +++ b/node/src/main.rs @@ -20,9 +20,7 @@ use graph_core::{ SubgraphRegistrar as IpfsSubgraphRegistrar, }; use graph_graphql::prelude::GraphQlRunner; -use graph_node::config::Config; use graph_node::network_setup::Networks; -use graph_node::opt; use graph_node::store_builder::StoreBuilder; use graph_server_http::GraphQLServer as GraphQLQueryServer; use graph_server_index_node::IndexNodeServer; @@ -32,6 +30,8 @@ use graph_server_websocket::SubscriptionServer as GraphQLSubscriptionServer; use graph_store_postgres::connection_pool::ConnectionPool; use graph_store_postgres::Store; use graph_store_postgres::{register_jobs as register_store_jobs, NotificationSender}; +use graphman::config::Config; +use graphman::opt; use graphman_server::GraphmanServer; use graphman_server::GraphmanServerConfig; use std::io::{BufRead, BufReader}; diff --git a/node/src/manager/commands/config.rs b/node/src/manager/commands/config.rs index 8b6d36e9afa..8c29641e1a0 100644 --- a/node/src/manager/commands/config.rs +++ b/node/src/manager/commands/config.rs @@ -16,7 +16,8 @@ use graph::{ use graph_chain_ethereum::NodeCapabilities; use graph_store_postgres::DeploymentPlacer; -use crate::{config::Config, network_setup::Networks}; +use crate::network_setup::Networks; +use graphman::config::Config; pub fn place(placer: &dyn DeploymentPlacer, name: &str, network: &str) -> Result<(), Error> { match placer.place(name, network).map_err(|s| anyhow!(s))? { diff --git a/node/src/manager/commands/config_cmd/check.rs b/node/src/manager/commands/config_cmd/check.rs new file mode 100644 index 00000000000..abd3563ea6a --- /dev/null +++ b/node/src/manager/commands/config_cmd/check.rs @@ -0,0 +1,26 @@ +use anyhow::Error; +use graphman::{commands::config::check::check, config::Config}; + +pub fn run(config: &Config, print: bool) -> Result<(), Error> { + let check = check(config, print); + + match check { + Ok(res) => { + if res.validated { + println!("Successfully validated configuration"); + } + if res.validated_subgraph_settings { + println!("Successfully validated subgraph settings"); + } + if let Some(txt) = res.config_json { + println!("{}", txt); + } + } + Err(e) => { + eprintln!("configuration error: {}", e); + std::process::exit(1); + } + } + + Ok(()) +} diff --git a/node/src/manager/commands/config_cmd/mod.rs b/node/src/manager/commands/config_cmd/mod.rs new file mode 100644 index 00000000000..db0b34e44da --- /dev/null +++ b/node/src/manager/commands/config_cmd/mod.rs @@ -0,0 +1,3 @@ +pub mod check; +pub mod place; +pub mod pools; diff --git a/node/src/manager/commands/config_cmd/place.rs b/node/src/manager/commands/config_cmd/place.rs new file mode 100644 index 00000000000..172d4b68257 --- /dev/null +++ b/node/src/manager/commands/config_cmd/place.rs @@ -0,0 +1,22 @@ +use anyhow::Error; +use graph_store_postgres::DeploymentPlacer; +use graphman::commands::config::place::{place, PlaceError}; + +pub fn run(placer: &dyn DeploymentPlacer, name: &String, network: &String) -> Result<(), Error> { + let res = place(placer, name, network); + match res { + Ok(result) => { + println!("subgraph: {}", name); + println!("network: {}", network); + println!("shard: {}", result.shards.join(", ")); + println!("nodes: {}", result.nodes.join(", ")); + } + Err(PlaceError::NoPlacementRule) => { + println!("{}", PlaceError::NoPlacementRule); + } + Err(PlaceError::Common(e)) => { + println!("Error: {}", e.to_string()); + } + }; + Ok(()) +} diff --git a/node/src/manager/commands/config_cmd/pools.rs b/node/src/manager/commands/config_cmd/pools.rs new file mode 100644 index 00000000000..d322e1ceaad --- /dev/null +++ b/node/src/manager/commands/config_cmd/pools.rs @@ -0,0 +1,22 @@ +use anyhow::Error; +use graphman::{commands::config::pools::pools, config::Config}; + +pub fn run(config: &Config, nodes: Vec, shard: bool) -> Result<(), Error> { + if nodes.is_empty() { + return Err(Error::msg("No nodes specified")); + } + let res = pools(config, &nodes)?; + if shard { + for shard in res.shards { + println!("{}: {}", shard.0, shard.1); + } + } else { + for pool in res.pools { + println!("{}:", pool.node); + for shard in pool.shards { + println!(" {}: {}", shard.0, shard.1); + } + } + } + Ok(()) +} diff --git a/node/src/manager/commands/mod.rs b/node/src/manager/commands/mod.rs index cb81a19ecb3..d15128b07e9 100644 --- a/node/src/manager/commands/mod.rs +++ b/node/src/manager/commands/mod.rs @@ -2,6 +2,7 @@ pub mod assign; pub mod chain; pub mod check_blocks; pub mod config; +pub mod config_cmd; pub mod copy; pub mod create; pub mod database; diff --git a/node/src/manager/commands/run.rs b/node/src/manager/commands/run.rs index 2c6bfdcb148..e724bd99a75 100644 --- a/node/src/manager/commands/run.rs +++ b/node/src/manager/commands/run.rs @@ -2,7 +2,6 @@ use std::collections::HashMap; use std::sync::Arc; use std::time::Duration; -use crate::config::Config; use crate::manager::PanicSubscriptionManager; use crate::network_setup::Networks; use crate::store_builder::StoreBuilder; @@ -26,6 +25,7 @@ use graph_core::{ SubgraphAssignmentProvider as IpfsSubgraphAssignmentProvider, SubgraphInstanceManager, SubgraphRegistrar as IpfsSubgraphRegistrar, }; +use graphman::config::Config; fn locate(store: &dyn SubgraphStore, hash: &str) -> Result { let mut locators = store.locators(hash)?; diff --git a/node/src/network_setup.rs b/node/src/network_setup.rs index 4a8b4cedca1..213c0a4bd5b 100644 --- a/node/src/network_setup.rs +++ b/node/src/network_setup.rs @@ -185,7 +185,7 @@ impl Networks { pub async fn from_config( logger: Logger, - config: &crate::config::Config, + config: &graphman::config::Config, registry: Arc, endpoint_metrics: Arc, provider_checks: &[Arc], diff --git a/node/src/store_builder.rs b/node/src/store_builder.rs index 2a39d0ea6ed..9deb1b72fc7 100644 --- a/node/src/store_builder.rs +++ b/node/src/store_builder.rs @@ -17,7 +17,7 @@ use graph_store_postgres::{ SubscriptionManager, PRIMARY_SHARD, }; -use crate::config::{Config, Shard}; +use graphman::config::{Config, Shard}; pub struct StoreBuilder { logger: Logger, diff --git a/server/graphman/Cargo.toml b/server/graphman/Cargo.toml index 231ef5e0828..ab8313bc4a3 100644 --- a/server/graphman/Cargo.toml +++ b/server/graphman/Cargo.toml @@ -9,6 +9,7 @@ async-graphql = { workspace = true } async-graphql-axum = { workspace = true } axum = { workspace = true } chrono = { workspace = true } +clap = { workspace = true } graph = { workspace = true } graph-store-postgres = { workspace = true } graphman = { workspace = true } diff --git a/server/graphman/src/entities/config_check.rs b/server/graphman/src/entities/config_check.rs new file mode 100644 index 00000000000..e25b32d6844 --- /dev/null +++ b/server/graphman/src/entities/config_check.rs @@ -0,0 +1,21 @@ +use async_graphql::SimpleObject; + +#[derive(Clone, Debug, SimpleObject)] +pub struct ConfigCheckResponse { + /// Checks if the config file is validated. + pub config_validated: bool, + /// Checks if the subgraph settings config set by GRAPH_EXPERIMENTAL_SUBGRAPH_SETTINGS are validated. + pub subgraph_settings_validated: bool, + /// Returns the Config file as a string. + pub config: String, +} + +impl ConfigCheckResponse { + pub fn from(config_validated: bool, subgraph_settings_validated: bool, config: String) -> Self { + Self { + config_validated, + subgraph_settings_validated, + config, + } + } +} diff --git a/server/graphman/src/entities/mod.rs b/server/graphman/src/entities/mod.rs index 8f4b2d8c018..2ff6a0e4e19 100644 --- a/server/graphman/src/entities/mod.rs +++ b/server/graphman/src/entities/mod.rs @@ -2,6 +2,7 @@ mod block_hash; mod block_number; mod block_ptr; mod command_kind; +mod config_check; mod deployment_info; mod deployment_selector; mod deployment_status; @@ -9,12 +10,15 @@ mod deployment_version_selector; mod empty_response; mod execution; mod execution_id; +mod place_response; +mod pools; mod subgraph_health; pub use self::block_hash::BlockHash; pub use self::block_number::BlockNumber; pub use self::block_ptr::BlockPtr; pub use self::command_kind::CommandKind; +pub use self::config_check::ConfigCheckResponse; pub use self::deployment_info::DeploymentInfo; pub use self::deployment_selector::DeploymentSelector; pub use self::deployment_status::DeploymentStatus; @@ -22,4 +26,6 @@ pub use self::deployment_version_selector::DeploymentVersionSelector; pub use self::empty_response::EmptyResponse; pub use self::execution::Execution; pub use self::execution_id::ExecutionId; +pub use self::place_response::PlaceResponse; +pub use self::pools::PoolsResponse; pub use self::subgraph_health::SubgraphHealth; diff --git a/server/graphman/src/entities/place_response.rs b/server/graphman/src/entities/place_response.rs new file mode 100644 index 00000000000..83d05db5545 --- /dev/null +++ b/server/graphman/src/entities/place_response.rs @@ -0,0 +1,25 @@ +use async_graphql::SimpleObject; + +#[derive(Clone, Debug, SimpleObject)] +pub struct PlaceResponse { + subgraph: String, + network: String, + shards: Vec, + nodes: Vec, +} + +impl PlaceResponse { + pub fn from( + subgraph: String, + network: String, + shards: Vec, + nodes: Vec, + ) -> Self { + Self { + subgraph, + network, + shards, + nodes, + } + } +} diff --git a/server/graphman/src/entities/pools.rs b/server/graphman/src/entities/pools.rs new file mode 100644 index 00000000000..ad394080ad2 --- /dev/null +++ b/server/graphman/src/entities/pools.rs @@ -0,0 +1,12 @@ +use std::collections::BTreeMap; + +use async_graphql::SimpleObject; +use graphman::commands::config::pools::Pools; + +#[derive(Clone, Debug, SimpleObject)] +pub struct PoolsResponse { + /// Size of database pools for each node. + pub pools: Vec, + /// Connections by shard rather than by node + pub shards: BTreeMap, +} diff --git a/server/graphman/src/resolvers/config_query.rs b/server/graphman/src/resolvers/config_query.rs new file mode 100644 index 00000000000..3ae316650fb --- /dev/null +++ b/server/graphman/src/resolvers/config_query.rs @@ -0,0 +1,36 @@ +use async_graphql::Object; +use async_graphql::Result; + +use crate::entities::ConfigCheckResponse; +use crate::entities::PlaceResponse; +use crate::entities::PoolsResponse; + +mod check; +mod place; +mod pools; +pub struct ConfigQuery; + +#[Object] +impl ConfigQuery { + /// Check and validate the configuration file + pub async fn check(&self) -> Result { + check::run() + } + + /// Returns how a specific subgraph would be placed + pub async fn place( + &self, + #[graphql(desc = "Subgraph name")] subgraph: String, + #[graphql(desc = "Network name")] network: String, + ) -> Result { + place::run(subgraph, network).await + } + + // Information about the size of database pools + pub async fn pools( + &self, + #[graphql(desc = "The names of the nodes that are going to run")] nodes: Vec, + ) -> Result { + pools::run(nodes).await + } +} diff --git a/server/graphman/src/resolvers/config_query/check.rs b/server/graphman/src/resolvers/config_query/check.rs new file mode 100644 index 00000000000..ae9d35f6040 --- /dev/null +++ b/server/graphman/src/resolvers/config_query/check.rs @@ -0,0 +1,68 @@ +use async_graphql::Result; +use clap::Parser; +use graph::{log::logger, prelude::error}; +use graphman::{commands::config::check::check, config::Config, opt::Opt}; + +use crate::entities::ConfigCheckResponse; + +pub fn run() -> Result { + let config = fetch_config()?; + let res = check(&config, true)?; + + Ok(ConfigCheckResponse::from( + res.validated, + res.validated_subgraph_settings, + res.config_json.unwrap_or_default(), + )) +} + +pub fn fetch_config() -> Result { + let args: Vec = std::env::args().collect(); + let accepted_flags = vec![ + "--config", + "--check-config", + "--subgraph", + "--start-block", + "--postgres-url", + "--postgres-secondary-hosts", + "--postgres-host-weights", + "--ethereum-rpc", + "--ethereum-ws", + "--ethereum-ipc", + "--ipfs", + "--arweave", + "--http-port", + "--index-node-port", + "--ws-port", + "--admin-port", + "--metrics-port", + "--node-id", + "--expensive-queries-filename", + "--debug", + "--elasticsearch-url", + "--elasticsearch-user", + "--elasticsearch-password", + "--disable-block-ingestor", + "--store-connection-pool-size", + "--unsafe-config", + "--debug-fork", + "--fork-base", + "--graphman-port", + ]; + let mut filtered_args: Vec = vec![args[0].clone()]; + for (i, arg) in args.iter().enumerate() { + if accepted_flags.contains(&arg.as_str()) { + filtered_args.push(arg.clone()); + filtered_args.push(args[i + 1].clone()); + } + } + let opt = Opt::try_parse_from(filtered_args).expect("Failed to parse args"); + let logger = logger(opt.debug); + match Config::load(&logger, &opt.clone().into()) { + Ok(config) => Ok(config), + Err(e) => { + error!(logger, "Failed to load config due to: {}", e.to_string()); + return Err(e.into()); + } + } +} diff --git a/server/graphman/src/resolvers/config_query/place.rs b/server/graphman/src/resolvers/config_query/place.rs new file mode 100644 index 00000000000..16ea2507ea4 --- /dev/null +++ b/server/graphman/src/resolvers/config_query/place.rs @@ -0,0 +1,15 @@ +use async_graphql::Result; +use graphman::commands::config::place::place; + +use crate::entities::PlaceResponse; + +use super::check::fetch_config; + +pub async fn run(subgraph: String, network: String) -> Result { + let config = fetch_config()?; + let res = place(&config.deployment, &subgraph, &network)?; + + Ok(PlaceResponse::from( + subgraph, network, res.shards, res.nodes, + )) +} diff --git a/server/graphman/src/resolvers/config_query/pools.rs b/server/graphman/src/resolvers/config_query/pools.rs new file mode 100644 index 00000000000..5e3f92a0734 --- /dev/null +++ b/server/graphman/src/resolvers/config_query/pools.rs @@ -0,0 +1,19 @@ +use async_graphql::Result; +use graphman::commands::config::pools::pools; + +use crate::entities::PoolsResponse; + +use super::check::fetch_config; + +pub async fn run(nodes: Vec) -> Result { + let config = fetch_config()?; + let res = pools(&config, &nodes); + + match res { + Ok(res) => Ok(PoolsResponse { + pools: res.pools, + shards: res.shards, + }), + Err(e) => Err(e.into()), + } +} diff --git a/server/graphman/src/resolvers/context.rs b/server/graphman/src/resolvers/context.rs index 8cc3e819c6d..e3e29ebef70 100644 --- a/server/graphman/src/resolvers/context.rs +++ b/server/graphman/src/resolvers/context.rs @@ -17,7 +17,6 @@ impl GraphmanContext { let primary_pool = ctx.data::()?.to_owned(); let notification_sender = ctx.data::>()?.to_owned(); let store = ctx.data::>()?.to_owned(); - Ok(GraphmanContext { primary_pool, notification_sender, diff --git a/server/graphman/src/resolvers/mod.rs b/server/graphman/src/resolvers/mod.rs index 2f7f225f6f4..f10854bd87b 100644 --- a/server/graphman/src/resolvers/mod.rs +++ b/server/graphman/src/resolvers/mod.rs @@ -1,3 +1,4 @@ +mod config_query; mod context; mod deployment_mutation; mod deployment_query; @@ -5,6 +6,7 @@ mod execution_query; mod mutation_root; mod query_root; +pub use self::config_query::ConfigQuery; pub use self::deployment_mutation::DeploymentMutation; pub use self::deployment_query::DeploymentQuery; pub use self::execution_query::ExecutionQuery; diff --git a/server/graphman/src/resolvers/query_root.rs b/server/graphman/src/resolvers/query_root.rs index 1c105abe40a..8ad3818280e 100644 --- a/server/graphman/src/resolvers/query_root.rs +++ b/server/graphman/src/resolvers/query_root.rs @@ -1,5 +1,6 @@ use async_graphql::Object; +use crate::resolvers::ConfigQuery; use crate::resolvers::DeploymentQuery; use crate::resolvers::ExecutionQuery; @@ -13,6 +14,11 @@ impl QueryRoot { DeploymentQuery {} } + /// Queries related to Config. + pub async fn config(&self) -> ConfigQuery { + ConfigQuery {} + } + /// Queries related to command executions. pub async fn execution(&self) -> ExecutionQuery { ExecutionQuery {} diff --git a/server/graphman/tests/config_query.rs b/server/graphman/tests/config_query.rs new file mode 100644 index 00000000000..3e19111bb04 --- /dev/null +++ b/server/graphman/tests/config_query.rs @@ -0,0 +1,283 @@ +pub mod util; + +use graph::log::logger; +use graphman::config::Config; +use serde_json::json; + +use self::util::client::send_graphql_request; +use self::util::run_test; +use self::util::server::VALID_TOKEN; + +#[test] +fn graphql_can_validate_config_and_subgraphs_settings_config() { + run_test(|| async { + let curr_dir = std::env::current_dir() + .unwrap() + .to_str() + .unwrap() + .to_string(); + let config_dir = format!("{}/tests/test_config.toml", curr_dir); + let subgraph_settings_dir = format!("{}/tests/test_subgraph_settings.toml", curr_dir); + std::env::set_var("GRAPH_NODE_CONFIG", config_dir); + std::env::set_var( + "GRAPH_EXPERIMENTAL_SUBGRAPH_SETTINGS", + subgraph_settings_dir, + ); + + let resp = send_graphql_request( + json!({ + "query": r#"{ + config { + check { + configValidated + subgraphSettingsValidated + } + } + }"# + }), + VALID_TOKEN, + ) + .await; + + let expected_resp = json!({ + "data": { + "config": { + "check": { + "configValidated": true, + "subgraphSettingsValidated": true + } + } + } + }); + + assert_eq!(resp, expected_resp); + }); +} + +#[test] +fn graphql_can_return_config_as_json_string() { + run_test(|| async { + let curr_dir = std::env::current_dir() + .unwrap() + .to_str() + .unwrap() + .to_string(); + let config_dir = format!("{}/tests/test_config.toml", curr_dir); + std::env::set_var("GRAPH_NODE_CONFIG", config_dir.clone()); + + let resp = send_graphql_request( + json!({ + "query": r#"{ + config { + check { + config + } + } + }"# + }), + VALID_TOKEN, + ) + .await; + + let config = Config::from_file(&logger(true), &config_dir, "default") + .unwrap() + .to_json() + .unwrap(); + + assert_eq!( + resp.get("data") + .unwrap() + .get("config") + .unwrap() + .get("check") + .unwrap() + .get("config") + .unwrap() + .as_str() + .unwrap(), + config + ); + }); +} + +#[test] +fn graphql_can_check_how_specific_subgraph_would_be_placed() { + run_test(|| async { + let curr_dir = std::env::current_dir() + .unwrap() + .to_str() + .unwrap() + .to_string(); + let config_dir = format!("{}/tests/test_config.toml", curr_dir); + std::env::set_var("GRAPH_NODE_CONFIG", config_dir); + + let resp = send_graphql_request( + json!({ + "query": r#"{ + config { + place(subgraph: "subgraph_1", network: "bsc") { + subgraph + network + shards + nodes + } + } + }"# + }), + VALID_TOKEN, + ) + .await; + + let expected_resp = json!({ + "data": { + "config": { + "place": { + "subgraph": String::from("subgraph_1"), + "network": String::from("bsc"), + "shards": vec!["primary".to_string(),"shard_a".to_string()], + "nodes": vec!["index_node_1_a".to_string(),"index_node_2_a".to_string(),"index_node_3_a".to_string()], + } + } + } + }); + + let resp_custom = send_graphql_request( + json!({ + "query": r#"{ + config { + place(subgraph: "custom/subgraph_1", network: "bsc") { + subgraph + network + shards + nodes + } + } + }"# + }), + VALID_TOKEN, + ) + .await; + + let expected_resp_custom = json!({ + "data": { + "config": { + "place": { + "subgraph": String::from("custom/subgraph_1"), + "network": String::from("bsc"), + "shards": vec!["primary".to_string()], + "nodes": vec!["index_custom_0".to_string()], + } + } + } + }); + + assert_eq!(resp, expected_resp); + assert_eq!(resp_custom, expected_resp_custom); + }); +} + +#[test] +fn graphql_can_fetch_info_about_size_of_database_pools() { + run_test(|| async { + let curr_dir = std::env::current_dir() + .unwrap() + .to_str() + .unwrap() + .to_string(); + let config_dir = format!("{}/tests/test_config.toml", curr_dir); + std::env::set_var("GRAPH_NODE_CONFIG", config_dir); + + let resp = send_graphql_request( + json!({ + "query": r#"{ + config { + pools(nodes: ["index_node_1_a", "index_node_2_a", "index_node_3_a"]) { + pools { + shards + node + } + } + } + }"# + }), + VALID_TOKEN, + ) + .await; + + let expected_resp = json!({ + "data": { + "config": { + "pools": { + "pools": [ + { + "shards": { + "primary": 2, + "shard_a": 2 + }, + "node": "index_node_1_a" + }, + { + "shards": { + "primary": 10, + "shard_a": 10 + }, + "node": "index_node_2_a" + }, + { + "shards": { + "primary": 10, + "shard_a": 10 + }, + "node": "index_node_3_a" + } + ] + } + } + } + }); + + assert_eq!(resp, expected_resp); + }); +} + +#[test] +fn graphql_can_fetch_connections_by_shard() { + run_test(|| async { + let curr_dir = std::env::current_dir() + .unwrap() + .to_str() + .unwrap() + .to_string(); + let config_dir = format!("{}/tests/test_config.toml", curr_dir); + std::env::set_var("GRAPH_NODE_CONFIG", config_dir); + + let resp = send_graphql_request( + json!({ + "query": r#"{ + config { + pools(nodes: ["index_node_1_a", "index_node_2_a", "index_node_3_a"]) { + shards + } + } + }"# + }), + VALID_TOKEN, + ) + .await; + + let expected_resp = json!({ + "data": { + "config": { + "pools": { + "shards": { + "primary": 22, + "shard_a": 22 + } + } + } + } + }); + + assert_eq!(resp, expected_resp); + }); +} diff --git a/server/graphman/tests/test_config.toml b/server/graphman/tests/test_config.toml new file mode 100644 index 00000000000..1f907539194 --- /dev/null +++ b/server/graphman/tests/test_config.toml @@ -0,0 +1,71 @@ +[general] +query = "query_node_.*" + +[store] +[store.primary] +connection = "postgresql://postgres:1.1.1.1@test/primary" +pool_size = [ + { node = "index_node_1_.*", size = 2 }, + { node = "index_node_2_.*", size = 10 }, + { node = "index_node_3_.*", size = 10 }, + { node = "index_node_4_.*", size = 2 }, + { node = "query_node_.*", size = 10 } +] + +[store.shard_a] +connection = "postgresql://postgres:1.1.1.1@test/shard-a" +pool_size = [ + { node = "index_node_1_.*", size = 2 }, + { node = "index_node_2_.*", size = 10 }, + { node = "index_node_3_.*", size = 10 }, + { node = "index_node_4_.*", size = 2 }, + { node = "query_node_.*", size = 10 } +] + +[deployment] +# Studio subgraphs +[[deployment.rule]] +match = { name = "^prefix/" } +shard = "shard_a" +indexers = [ "index_prefix_0", + "index_prefix_1" ] + +[[deployment.rule]] +match = { name = "^custom/.*" } +indexers = [ "index_custom_0" ] + +[[deployment.rule]] +shards = [ "primary", "shard_a" ] +indexers = [ "index_node_1_a", + "index_node_2_a", + "index_node_3_a" ] + +[chains] +ingestor = "index_0" + +[chains.mainnet] +shard = "primary" +provider = [ + { label = "mainnet-0", url = "http://rpc.mainnet.io", features = ["archive", "traces"] }, + { label = "mainnet-1", details = { type = "web3call", url = "http://rpc.mainnet.io", features = ["archive", "traces"] }}, + { label = "firehose", details = { type = "firehose", url = "http://localhost:9000", features = [] }}, + { label = "substreams", details = { type = "substreams", url = "http://localhost:9000", features = [] }}, +] + +[chains.ropsten] +shard = "primary" +provider = [ + { label = "ropsten-0", url = "http://rpc.ropsten.io", transport = "rpc", features = ["archive", "traces"] } +] + +[chains.goerli] +shard = "primary" +provider = [ + { label = "goerli-0", url = "http://rpc.goerli.io", transport = "ipc", features = ["archive"] } +] + +[chains.kovan] +shard = "primary" +provider = [ + { label = "kovan-0", url = "http://rpc.kovan.io", transport = "ws", features = [] } +] diff --git a/server/graphman/tests/test_subgraph_settings.toml b/server/graphman/tests/test_subgraph_settings.toml new file mode 100644 index 00000000000..86a500c037b --- /dev/null +++ b/server/graphman/tests/test_subgraph_settings.toml @@ -0,0 +1,11 @@ +[[setting]] +match = { name = ".*" } +history_blocks = 10000 + +[[setting]] +match = { name = "xxxxx" } +history_blocks = 10000 + +[[setting]] +match = { name = ".*!$" } +history_blocks = 10000 diff --git a/store/test-store/Cargo.toml b/store/test-store/Cargo.toml index fe05f12233e..89dcd0ce25c 100644 --- a/store/test-store/Cargo.toml +++ b/store/test-store/Cargo.toml @@ -15,6 +15,7 @@ lazy_static = "1.5" hex-literal = "0.4" diesel = { workspace = true } prost-types = { workspace = true } +graphman = { workspace = true } [dev-dependencies] hex = "0.4.3" diff --git a/store/test-store/src/store.rs b/store/test-store/src/store.rs index afb088f6bf6..ad289841ec3 100644 --- a/store/test-store/src/store.rs +++ b/store/test-store/src/store.rs @@ -23,7 +23,6 @@ use graph_graphql::prelude::{ execute_query, Query as PreparedQuery, QueryExecutionOptions, StoreResolver, }; use graph_graphql::test_support::GraphQLMetrics; -use graph_node::config::{Config, Opt}; use graph_node::store_builder::StoreBuilder; use graph_store_postgres::layout_for_tests::FAKE_NETWORK_SHARED; use graph_store_postgres::{connection_pool::ConnectionPool, Shard, SubscriptionManager}; @@ -31,6 +30,7 @@ use graph_store_postgres::{ BlockStore as DieselBlockStore, DeploymentPlacer, SubgraphStore as DieselSubgraphStore, PRIMARY_SHARD, }; +use graphman::config::{Config, Opt}; use hex_literal::hex; use lazy_static::lazy_static; use std::collections::BTreeSet; diff --git a/tests/Cargo.toml b/tests/Cargo.toml index b79702ceb7f..f350b295d75 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -9,16 +9,20 @@ assert-json-diff = "2.0.2" async-stream = "0.3.5" graph = { path = "../graph" } graph-chain-ethereum = { path = "../chain/ethereum" } -graph-chain-substreams= {path = "../chain/substreams"} +graph-chain-substreams = { path = "../chain/substreams" } graph-node = { path = "../node" } graph-core = { path = "../core" } graph-graphql = { path = "../graphql" } graph-store-postgres = { path = "../store/postgres" } graph-server-index-node = { path = "../server/index-node" } graph-runtime-wasm = { path = "../runtime/wasm" } +graphman = { workspace = true } serde = { workspace = true } serde_yaml = { workspace = true } -slog = { version = "2.7.0", features = ["release_max_level_trace", "max_level_trace"] } +slog = { version = "2.7.0", features = [ + "release_max_level_trace", + "max_level_trace", +] } tokio = { version = "1.38.0", features = ["rt", "macros", "process"] } # Once graph upgrades to web3 0.19, we don't need this anymore. The version # here needs to be kept in sync with the web3 version that the graph crate diff --git a/tests/src/fixture/mod.rs b/tests/src/fixture/mod.rs index 4eb5fbb42b1..d1d021fc9c2 100644 --- a/tests/src/fixture/mod.rs +++ b/tests/src/fixture/mod.rs @@ -51,10 +51,11 @@ use graph_core::{ SubgraphRegistrar as IpfsSubgraphRegistrar, SubgraphTriggerProcessor, }; use graph_node::manager::PanicSubscriptionManager; -use graph_node::{config::Config, store_builder::StoreBuilder}; +use graph_node::store_builder::StoreBuilder; use graph_runtime_wasm::RuntimeHostBuilder; use graph_server_index_node::IndexNodeService; use graph_store_postgres::{ChainHeadUpdateListener, ChainStore, Store, SubgraphStore}; +use graphman::config::Config; use serde::Deserialize; use slog::{crit, debug, info, o, Discard, Logger}; use std::env::VarError;