From 978c446664b3853307daa13ebf3c376818e1de5a Mon Sep 17 00:00:00 2001 From: Greg Cusack Date: Tue, 6 Aug 2024 16:43:57 -0700 Subject: [PATCH] Add support for generic clients (#49) * wip. parsing of generic arguments is weird and not working * fixed arg parsing. next add shred_version and other validator ENV variables * finish up generic client * address comments: update client trait, propagate errors up --- PROGRESS.md | 1 + README.md | 122 +++++++++++- src/client_config.rs | 115 ++++++++++- src/cluster_images.rs | 38 ++-- src/docker.rs | 142 +++++++------ src/genesis.rs | 34 ++-- src/k8s_helpers.rs | 2 +- src/kubernetes.rs | 68 ++----- src/lib.rs | 36 +++- src/main.rs | 361 ++++++++++++++++++++++------------ src/{validator.rs => node.rs} | 14 +- 11 files changed, 638 insertions(+), 295 deletions(-) rename src/{validator.rs => node.rs} (89%) diff --git a/PROGRESS.md b/PROGRESS.md index ab6273b..e3414bf 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -82,6 +82,7 @@ By here: - [ ] Other Features - [x] Heterogeneous Clusters (i.e. multiple validator versions) - [x] Deploy specific commit + - [x] Generic Clients - [ ] Deploy with user-defined stake distribution By here: diff --git a/README.md b/README.md index 2783b04..f3fef73 100644 --- a/README.md +++ b/README.md @@ -177,6 +177,92 @@ curl -X POST \ }' \ http://: ``` +Note: you can deploy any client through validator-lab or just completely separately and have the client send TXs or query this RPC through the `http://:`. + +## Generic Clients +Bring your own client and deploy it in a Validator Lab cluster! +All you need is a containerized version of your client in an accessible docker registry. + +Key points/steps: +1) [Containerize your client](#Containerize-your-Client) +2) Any client accounts should be built into the client container image +3) Client arguments are passed in similar to how they are passed into the bench-tps client. For the generic client, use `--generic-client-args`. + +For example, let's assume we have a client sending spam. And it takes the following arguments: +``` +/home/solana/spammer-executable --target-node : --thread-sleep-ms --spam-mode +``` +where `:` is the domain name and port of the kubernetes service running the validator you want to target. See: [Node Naming Conventions](#kubernetes_domain_name) + +When we go to deploy the generic client, we deploy it in a similar manner to how we deploy the bench-tps client: +``` +cargo run --bin cluster -- -n +... +generic-client --docker-image --executable-path --delay-start --generic-client-args 'target-node=: thread-sleep-ms= spam-mode=' +``` + +4) Any flag or value the client needs that is cluster specific should be read in from an environment variable. For example, say the client requires the following arguments: +``` +/home/solana/spammer-executable --target-node : --shred-version +``` +Shred-version is cluster specific; it is not known when you deploy a cluster. Modify the shred-version argument in the client code to read in the environment variable `SHRED_VERSION` from the host. +Example: +``` +let default_shred_version = env::var("SHRED_VERSION").unwrap_or_else(|_| "0".to_string()); +... +.arg( + Arg::with_name("shred_version") + .long("shred-version") + .takes_value(true) + .default_value(&default_shred_version) + .help("Shred version of cluster to spam"), +) +... +``` +When you deploy a cluster with your client, leave the `--shred-version` command out since it will be read via environment variable: +``` +cargo run --bin cluster -- -n +... +generic-client --docker-image --executable-path --delay-start --generic-client-args 'target-node=' +``` + +The following environment variables are available to each non-bootstrap pod: +``` +NAMESPACE # cluster namespace +BOOTSTRAP_RPC_ADDRESS # rpc address of bootstrap node +BOOTSTRAP_GOSSIP_ADDRESS # gossip address of bootstrap node +BOOTSTRAP_FAUCET_ADDRESS # faucet address of bootstrap node +SHRED_VERSION # cluster shred version +``` +^ More environment variables to come! + + +### Node Naming Conventions in Kubernetes +Say you want to launch your client and send transactions to a specific validator. Kubernetes makes it easy to identify deployed nodes via `:`. Node naming conventions: +``` +-service..svc.cluster.local: +``` +e.g. bootstrap validator RPC port can be reached with: +``` +bootstrap-validator-service..svc.cluster.local:8899 +``` +and a standard validator can be reached with: +``` +validator-service-<8-char-commit-or-version>-..svc.cluster.local: +``` +examples: +``` +# w/ commit +validator-service-bd1a5dfb-7.greg.svc.cluster.local:8001 +# or with version +validator-service-v1.18.16-4.greg.svc.cluster.local:8001 +``` +Say you want to deploy your client with `--target-node ` which is running v1.18.16: +``` +cargo run --bin cluster -- -n +... +generic-client --docker-image /: --executable-path --delay-start --generic-client-args 'target-node=validator-service-v1.18.16-4.greg.svc.cluster.local:8001' +``` ## Kubernetes Cheatsheet Create namespace: @@ -207,4 +293,38 @@ kubectl exec -it -n -- /bin/bash Get information about pod: ``` kubectl describe pod -n -``` \ No newline at end of file +``` + +## Containerize your Client +### Dockerfile Template +``` +FROM ubuntu:22.04 +RUN apt-get update && apt-get install -y iputils-ping curl vim && \ + rm -rf /var/lib/apt/lists/* && \ + useradd -ms /bin/bash solana && \ + adduser solana sudo + +USER solana +COPY --chown=solana:solana ./target/release/ /home/solana/ +COPY --chown=solana:solana ./client-accounts/ /home/solana/client-accounts/ +RUN chmod +x /home/solana/ +WORKDIR /home/solana +``` + +### Build client image +``` +cd +docker build -t /: -f /Dockerfile + +# e.g. +cd client-spam/ +docker build -t test-registry/client-spam:latest -f docker/Dockerfile . +``` + +### Push client image to registry +``` +docker push /: + +# e.g. +docker push test-registry/client-spam:latest +``` diff --git a/src/client_config.rs b/src/client_config.rs index e38ca98..be9aaaf 100644 --- a/src/client_config.rs +++ b/src/client_config.rs @@ -1,12 +1,117 @@ -use solana_sdk::pubkey::Pubkey; +use { + solana_sdk::pubkey::Pubkey, + std::{error::Error, path::PathBuf}, + strum_macros::Display, +}; -#[derive(Clone, Debug)] -pub struct ClientConfig { +#[derive(Clone, PartialEq, Debug)] +pub struct BenchTpsConfig { pub num_clients: usize, + pub client_duration_seconds: u64, pub client_type: String, - pub client_to_run: String, pub bench_tps_args: Vec, + pub client_wait_for_n_nodes: Option, + pub client_to_run: String, pub client_target_node: Option, +} + +impl ClientTrait for BenchTpsConfig { + fn executable_path(&self) -> Result, Box> { + let command = vec!["/home/solana/k8s-cluster-scripts/client-startup-script.sh".to_string()]; + Ok(command) + } + + fn generate_client_command_flags(&self) -> Vec { + let mut flags = vec![]; + + flags.push(self.client_to_run.clone()); //client to run + if !self.bench_tps_args.is_empty() { + flags.push(self.bench_tps_args.join(" ")); + } + + flags.push(self.client_type.clone()); + + if let Some(target_node) = self.client_target_node { + flags.push("--target-node".to_string()); + flags.push(target_node.to_string()); + } + + flags.push("--duration".to_string()); + flags.push(self.client_duration_seconds.to_string()); + + if let Some(num_nodes) = self.client_wait_for_n_nodes { + flags.push("--num-nodes".to_string()); + flags.push(num_nodes.to_string()); + } + + flags + } +} + +#[derive(Default, Clone, PartialEq, Debug)] +pub struct GenericClientConfig { + pub num_clients: usize, pub client_duration_seconds: u64, - pub client_wait_for_n_nodes: Option, + pub args: Vec, + pub image: String, + pub executable_path: PathBuf, + pub delay_start: u64, +} + +impl ClientTrait for GenericClientConfig { + fn executable_path(&self) -> Result, Box> { + let exec_path_string = self + .executable_path + .clone() + .into_os_string() + .into_string() + .map_err(|err| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("Invalid Unicode data in path: {:?}", err), + ) + })?; + Ok(vec![exec_path_string]) + } + fn generate_client_command_flags(&self) -> Vec { + self.args.clone() + } +} + +#[derive(Debug, Clone, PartialEq, Display)] +pub enum ClientConfig { + #[strum(serialize = "bench-tps")] + BenchTps(BenchTpsConfig), + #[strum(serialize = "generic")] + Generic(GenericClientConfig), + None, +} + +impl ClientConfig { + pub fn num_clients(&self) -> usize { + match self { + ClientConfig::BenchTps(config) => config.num_clients, + ClientConfig::Generic(config) => config.num_clients, + ClientConfig::None => 0, + } + } + + pub fn build_command(&self) -> Result, Box> { + match self { + ClientConfig::BenchTps(config) => config.build_command(), + ClientConfig::Generic(config) => config.build_command(), + ClientConfig::None => Err("Client config is None".into()), + } + } +} + +pub trait ClientTrait { + fn executable_path(&self) -> Result, Box>; + fn generate_client_command_flags(&self) -> Vec; + /// Build command to run on pod deployment + fn build_command(&self) -> Result, Box> { + let mut command = self.executable_path()?; + command.extend(self.generate_client_command_flags()); + Ok(command) + } } diff --git a/src/cluster_images.rs b/src/cluster_images.rs index 1e7b100..22b0d9f 100644 --- a/src/cluster_images.rs +++ b/src/cluster_images.rs @@ -1,5 +1,5 @@ use { - crate::{validator::Validator, ValidatorType}, + crate::{node::Node, NodeType}, std::{error::Error, result::Result}, }; @@ -11,41 +11,41 @@ use { #[derive(Default)] pub struct ClusterImages { - bootstrap: Option, - validator: Option, - rpc: Option, - clients: Vec, + bootstrap: Option, + validator: Option, + rpc: Option, + clients: Vec, } impl ClusterImages { - pub fn set_item(&mut self, item: Validator, validator_type: ValidatorType) { - match validator_type { - ValidatorType::Bootstrap => self.bootstrap = Some(item), - ValidatorType::Standard => self.validator = Some(item), - ValidatorType::RPC => self.rpc = Some(item), - ValidatorType::Client(_) => self.clients.push(item), + pub fn set_item(&mut self, item: Node) { + match item.node_type() { + NodeType::Bootstrap => self.bootstrap = Some(item), + NodeType::Standard => self.validator = Some(item), + NodeType::RPC => self.rpc = Some(item), + NodeType::Client(_, _) => self.clients.push(item), } } - pub fn bootstrap(&mut self) -> Result<&mut Validator, Box> { + pub fn bootstrap(&mut self) -> Result<&mut Node, Box> { self.bootstrap .as_mut() .ok_or_else(|| "Bootstrap validator is not available".into()) } - pub fn validator(&mut self) -> Result<&mut Validator, Box> { + pub fn validator(&mut self) -> Result<&mut Node, Box> { self.validator .as_mut() .ok_or_else(|| "Validator is not available".into()) } - pub fn rpc(&mut self) -> Result<&mut Validator, Box> { + pub fn rpc(&mut self) -> Result<&mut Node, Box> { self.rpc .as_mut() .ok_or_else(|| "Validator is not available".into()) } - pub fn client(&mut self, client_index: usize) -> Result<&mut Validator, Box> { + pub fn client(&mut self, client_index: usize) -> Result<&mut Node, Box> { if self.clients.is_empty() { return Err("No Clients available".to_string().into()); } @@ -54,7 +54,7 @@ impl ClusterImages { .ok_or_else(|| "Client index out of bounds".to_string().into()) } - pub fn get_validators(&self) -> impl Iterator { + pub fn get_validators(&self) -> impl Iterator { self.bootstrap .iter() .chain(self.validator.iter()) @@ -62,15 +62,15 @@ impl ClusterImages { .filter_map(Some) } - pub fn get_clients(&self) -> impl Iterator { + pub fn get_clients(&self) -> impl Iterator { self.clients.iter() } - pub fn get_clients_mut(&mut self) -> impl Iterator { + pub fn get_clients_mut(&mut self) -> impl Iterator { self.clients.iter_mut() } - pub fn get_all(&self) -> impl Iterator { + pub fn get_all(&self) -> impl Iterator { self.get_validators().chain(self.get_clients()) } } diff --git a/src/docker.rs b/src/docker.rs index 7fb7a12..98ff792 100644 --- a/src/docker.rs +++ b/src/docker.rs @@ -1,7 +1,7 @@ use { crate::{ - new_spinner_progress_bar, startup_scripts::StartupScripts, validator::Validator, - ValidatorType, BUILD, ROCKET, SOLANA_RELEASE, + new_spinner_progress_bar, node::Node, startup_scripts::StartupScripts, ClientType, + NodeType, BUILD, ROCKET, SOLANA_RELEASE, }, log::*, std::{ @@ -16,29 +16,50 @@ use { #[derive(Clone)] pub struct DockerImage { registry: String, - validator_type: ValidatorType, + node_type: NodeType, image_name: String, tag: String, // commit (`abcd1234`) or version (`v1.18.12`) + optional_full_image_path: Option, // /: } impl DockerImage { // Constructor to create a new instance of DockerImage - pub fn new( - registry: String, - validator_type: ValidatorType, - image_name: String, - tag: String, - ) -> Self { + pub fn new(registry: String, node_type: NodeType, image_name: String, tag: String) -> Self { DockerImage { registry, - validator_type, + node_type, image_name, tag, + optional_full_image_path: None, } } - pub fn validator_type(&self) -> ValidatorType { - self.validator_type + /// parse from string /: + pub fn new_from_string(image_string: String) -> Result> { + let split_string: Vec<&str> = image_string.split('/').collect(); + if split_string.len() != 2 { + return Err("Invalid format. Expected /:".into()); + } + + let registry = split_string[0].to_string(); + + // Split the second part into name and tag + let name_tag: Vec<&str> = split_string[1].split(':').collect(); + if name_tag.len() != 2 { + return Err("Invalid format. Expected /:".into()); + } + + Ok(DockerImage { + registry, + node_type: NodeType::Client(ClientType::Generic, 0), + image_name: name_tag[0].to_string(), + tag: name_tag[1].to_string(), + optional_full_image_path: Some(image_string), + }) + } + + pub fn node_type(&self) -> NodeType { + self.node_type } pub fn tag(&self) -> String { @@ -49,16 +70,22 @@ impl DockerImage { // Put DockerImage in format for building, pushing, and pulling impl Display for DockerImage { fn fmt(&self, f: &mut Formatter) -> fmt::Result { - match self.validator_type { - ValidatorType::Client(index) => write!( - f, - "{}/{}-{}-{}:{}", - self.registry, self.validator_type, index, self.image_name, self.tag - ), - ValidatorType::Bootstrap | ValidatorType::Standard | ValidatorType::RPC => write!( + match self.node_type { + NodeType::Client(_, index) => { + if let Some(image_path) = &self.optional_full_image_path { + write!(f, "{image_path}") + } else { + write!( + f, + "{}/{}-{}-{}:{}", + self.registry, self.node_type, index, self.image_name, self.tag + ) + } + } + NodeType::Bootstrap | NodeType::Standard | NodeType::RPC => write!( f, "{}/{}-{}:{}", - self.registry, self.validator_type, self.image_name, self.tag + self.registry, self.node_type, self.image_name, self.tag ), } } @@ -78,22 +105,17 @@ impl DockerConfig { solana_root_path: &Path, docker_image: &DockerImage, ) -> Result<(), Box> { - let validator_type = docker_image.validator_type(); - let docker_path = match validator_type { - ValidatorType::Bootstrap | ValidatorType::Standard | ValidatorType::RPC => { - solana_root_path.join(format!("docker-build/{validator_type}")) + let node_type = docker_image.node_type(); + let docker_path = match node_type { + NodeType::Bootstrap | NodeType::Standard | NodeType::RPC => { + solana_root_path.join(format!("docker-build/{node_type}")) } - ValidatorType::Client(index) => { - solana_root_path.join(format!("docker-build/{validator_type}-{index}")) + NodeType::Client(_, index) => { + solana_root_path.join(format!("docker-build/{node_type}-{index}")) } }; - self.create_base_image( - solana_root_path, - docker_image, - &docker_path, - &validator_type, - )?; + self.create_base_image(solana_root_path, docker_image, &docker_path, &node_type)?; Ok(()) } @@ -103,9 +125,9 @@ impl DockerConfig { solana_root_path: &Path, docker_image: &DockerImage, docker_path: &PathBuf, - validator_type: &ValidatorType, + node_type: &NodeType, ) -> Result<(), Box> { - self.create_dockerfile(validator_type, docker_path, solana_root_path, None)?; + self.create_dockerfile(node_type, docker_path, solana_root_path, None)?; // We use std::process::Command here because Docker-rs is very slow building dockerfiles // when they are in large repos. Docker-rs doesn't seem to support the `--file` flag natively. @@ -114,12 +136,12 @@ impl DockerConfig { let context_path = solana_root_path.display().to_string(); let progress_bar = new_spinner_progress_bar(); - progress_bar.set_message(format!("{BUILD}Building {validator_type} docker image...",)); - + progress_bar.set_message(format!("{BUILD}Building {node_type} docker image...",)); let command = format!( "docker build -t {docker_image} -f {} {context_path}", dockerfile.display() ); + debug!("docker command: {command}"); let output = Command::new("sh") .arg("-c") @@ -140,16 +162,16 @@ impl DockerConfig { fn write_startup_script_to_docker_directory( file_name: &str, docker_dir: &Path, - validator_type: &ValidatorType, + node_type: &NodeType, ) -> Result<(), Box> { let script_path = docker_dir.join(file_name); - let script_content = validator_type.script(); + let script_content = node_type.script(); StartupScripts::write_script_to_file(script_content, &script_path).map_err(|e| e.into()) } fn create_dockerfile( &self, - validator_type: &ValidatorType, + node_type: &NodeType, docker_path: &PathBuf, solana_root_path: &Path, content: Option<&str>, @@ -159,18 +181,18 @@ impl DockerConfig { } fs::create_dir_all(docker_path)?; - let file_name = format!("{validator_type}-startup-script.sh"); - Self::write_startup_script_to_docker_directory(&file_name, docker_path, validator_type)?; + let file_name = format!("{node_type}-startup-script.sh"); + Self::write_startup_script_to_docker_directory(&file_name, docker_path, node_type)?; StartupScripts::write_script_to_file( StartupScripts::common(), &docker_path.join("common.sh"), )?; - let startup_script_directory = match validator_type { - ValidatorType::Bootstrap | ValidatorType::Standard | ValidatorType::RPC => { - format!("./docker-build/{validator_type}") + let startup_script_directory = match node_type { + NodeType::Bootstrap | NodeType::Standard | NodeType::RPC => { + format!("./docker-build/{node_type}") } - ValidatorType::Client(index) => format!("./docker-build/{validator_type}-{index}"), + NodeType::Client(_, index) => format!("./docker-build/{node_type}-{index}"), }; let dockerfile = format!( @@ -193,8 +215,8 @@ WORKDIR /home/solana {} "#, self.base_image, - DockerConfig::check_copy_ledger(validator_type), - self.insert_client_accounts_if_present(solana_root_path, validator_type)? + DockerConfig::check_copy_ledger(node_type), + self.insert_client_accounts_if_present(solana_root_path, node_type)? ); debug!("dockerfile: {dockerfile:?}"); @@ -205,23 +227,23 @@ WORKDIR /home/solana Ok(()) } - fn check_copy_ledger(validator_type: &ValidatorType) -> String { - match validator_type { - ValidatorType::Bootstrap | ValidatorType::RPC => { + fn check_copy_ledger(node_type: &NodeType) -> String { + match node_type { + NodeType::Bootstrap | NodeType::RPC => { "COPY --chown=solana:solana ./config-k8s/bootstrap-validator /home/solana/ledger" .to_string() } - ValidatorType::Standard | &ValidatorType::Client(_) => "".to_string(), + NodeType::Standard | &NodeType::Client(_, _) => "".to_string(), } } fn insert_client_accounts_if_present( &self, solana_root_path: &Path, - validator_type: &ValidatorType, + node_type: &NodeType, ) -> Result> { - match validator_type { - ValidatorType::Client(index) => { + match node_type { + NodeType::Client(_, index) => { let bench_tps_path = solana_root_path.join(format!("config-k8s/bench-tps-{index}.yml")); if bench_tps_path.exists() { @@ -234,9 +256,7 @@ COPY --chown=solana:solana ./config-k8s/bench-tps-{index}.yml /home/solana/clien Err(format!("{bench_tps_path:?} does not exist!").into()) } } - ValidatorType::Bootstrap | ValidatorType::Standard | ValidatorType::RPC => { - Ok("".to_string()) - } + NodeType::Bootstrap | NodeType::Standard | NodeType::RPC => Ok("".to_string()), } } @@ -252,14 +272,14 @@ COPY --chown=solana:solana ./config-k8s/bench-tps-{index}.yml /home/solana/clien Ok(child) } - pub fn push_images<'a, I>(&self, validators: I) -> Result<(), Box> + pub fn push_images<'a, I>(&self, nodes: I) -> Result<(), Box> where - I: IntoIterator, + I: IntoIterator, { info!("Pushing images..."); - let children: Result, _> = validators + let children: Result, _> = nodes .into_iter() - .map(|validator| Self::push_image(validator.image())) + .map(|node| Self::push_image(node.image())) .collect(); let progress_bar = new_spinner_progress_bar(); diff --git a/src/genesis.rs b/src/genesis.rs index 8d403f8..a5fd9a9 100644 --- a/src/genesis.rs +++ b/src/genesis.rs @@ -1,5 +1,5 @@ use { - crate::{fetch_spl, new_spinner_progress_bar, ValidatorType, SOLANA_RELEASE, SUN, WRITING}, + crate::{fetch_spl, new_spinner_progress_bar, NodeType, SOLANA_RELEASE, SUN, WRITING}, log::*, rand::Rng, solana_core::gen_keys::GenKeys, @@ -155,20 +155,20 @@ impl Genesis { pub fn generate_accounts( &mut self, - validator_type: ValidatorType, + node_type: NodeType, number_of_accounts: usize, deployment_tag: Option<&str>, ) -> Result<(), Box> { - info!("generating {number_of_accounts} {validator_type} accounts..."); + info!("generating {number_of_accounts} {node_type} accounts..."); - let account_types = match validator_type { - ValidatorType::Bootstrap | ValidatorType::Standard => { + let account_types = match node_type { + NodeType::Bootstrap | NodeType::Standard => { vec!["identity", "stake-account", "vote-account"] } - ValidatorType::RPC => { + NodeType::RPC => { vec!["identity"] // no vote or stake account for RPC } - ValidatorType::Client(_) => { + NodeType::Client(_, _) => { return Err("Client valdiator_type in generate_accounts not allowed".into()) } }; @@ -190,28 +190,28 @@ impl Genesis { .key_generator .gen_n_keypairs(total_accounts_to_generate as u64); - self.write_accounts_to_file(&validator_type, &account_types, &keypairs)?; + self.write_accounts_to_file(&node_type, &account_types, &keypairs)?; Ok(()) } fn write_accounts_to_file( &self, - validator_type: &ValidatorType, + node_type: &NodeType, account_types: &[String], keypairs: &[Keypair], ) -> Result<(), Box> { for (i, keypair) in keypairs.iter().enumerate() { let account_index = i / account_types.len(); let account = &account_types[i % account_types.len()]; - let filename = match validator_type { - ValidatorType::Bootstrap => { - format!("{validator_type}/{account}.json") + let filename = match node_type { + NodeType::Bootstrap => { + format!("{node_type}/{account}.json") } - ValidatorType::Standard | ValidatorType::RPC => { - format!("{validator_type}-{account}-{account_index}.json") + NodeType::Standard | NodeType::RPC => { + format!("{node_type}-{account}-{account_index}.json") } - ValidatorType::Client(_) => panic!("Client type not supported"), + NodeType::Client(_, _) => panic!("Client type not supported"), }; let outfile = self.config_dir.join(&filename); @@ -253,7 +253,7 @@ impl Genesis { for child in children? { let output = child.wait_with_output()?; if !output.status.success() { - return Err(output.status.to_string().into()); + return Err(String::from_utf8_lossy(&output.stderr).into()); } } @@ -301,7 +301,7 @@ impl Genesis { let child = Command::new(executable_path) .args(args) .stdout(Stdio::null()) - .stderr(Stdio::null()) + .stderr(Stdio::piped()) .spawn()?; Ok(child) diff --git a/src/k8s_helpers.rs b/src/k8s_helpers.rs index 4755462..9d6b7cf 100644 --- a/src/k8s_helpers.rs +++ b/src/k8s_helpers.rs @@ -77,7 +77,7 @@ pub fn create_replica_set( }), spec: Some(PodSpec { containers: vec![Container { - name: format!("{}-container", image_name.validator_type()), + name: format!("{}-container", image_name.node_type()), image: Some(image_name.to_string()), image_pull_policy: Some("Always".to_string()), env: Some(environment_variables), diff --git a/src/kubernetes.rs b/src/kubernetes.rs index 8aaa52d..70ca068 100644 --- a/src/kubernetes.rs +++ b/src/kubernetes.rs @@ -4,7 +4,7 @@ use { docker::DockerImage, k8s_helpers::{self, SecretType}, validator_config::ValidatorConfig, - Metrics, ValidatorType, + Metrics, NodeType, }, k8s_openapi::{ api::{ @@ -294,13 +294,13 @@ impl<'a> Kubernetes<'a> { let command_path = format!( "/home/solana/k8s-cluster-scripts/{}-startup-script.sh", - ValidatorType::Bootstrap + NodeType::Bootstrap ); let mut command = vec![command_path]; command.extend(self.generate_bootstrap_command_flags()); k8s_helpers::create_replica_set( - format!("{}-{}", image.validator_type(), image.tag()), + format!("{}-{}", image.node_type(), image.tag()), self.namespace.clone(), label_selector.clone(), image.clone(), @@ -349,32 +349,6 @@ impl<'a> Kubernetes<'a> { flags } - fn generate_client_command_flags(&self) -> Vec { - let mut flags = vec![]; - - flags.push(self.client_config.client_to_run.clone()); //client to run - if !self.client_config.bench_tps_args.is_empty() { - flags.push(self.client_config.bench_tps_args.join(" ")); - } - - flags.push(self.client_config.client_type.clone()); - - if let Some(target_node) = self.client_config.client_target_node { - flags.push("--target-node".to_string()); - flags.push(target_node.to_string()); - } - - flags.push("--duration".to_string()); - flags.push(self.client_config.client_duration_seconds.to_string()); - - if let Some(num_nodes) = self.client_config.client_wait_for_n_nodes { - flags.push("--num-nodes".to_string()); - flags.push(num_nodes.to_string()); - } - - flags - } - pub fn create_selector(&self, key: &str, value: &str) -> BTreeMap { k8s_helpers::create_selector(key, value) } @@ -504,7 +478,7 @@ impl<'a> Kubernetes<'a> { } fn set_non_bootstrap_environment_variables(&self) -> Vec { - vec![ + let mut env_vars = vec![ k8s_helpers::create_environment_variable( "BOOTSTRAP_RPC_ADDRESS".to_string(), Some("bootstrap-validator-service.$(NAMESPACE).svc.cluster.local:8899".to_string()), @@ -520,7 +494,17 @@ impl<'a> Kubernetes<'a> { Some("bootstrap-validator-service.$(NAMESPACE).svc.cluster.local:9900".to_string()), None, ), - ] + ]; + if let Some(shred_version) = self.validator_config.shred_version { + let s = k8s_helpers::create_environment_variable( + "SHRED_VERSION".to_string(), + Some(shred_version.to_string()), + None, + ); + env_vars.push(s); + } + + env_vars } fn set_load_balancer_environment_variables(&self) -> Vec { @@ -619,12 +603,7 @@ impl<'a> Kubernetes<'a> { command.extend(self.generate_validator_command_flags()); k8s_helpers::create_replica_set( - format!( - "{}-{}-{}", - image.validator_type(), - image.tag(), - validator_index - ), + format!("{}-{}-{}", image.node_type(), image.tag(), validator_index), self.namespace.clone(), label_selector.clone(), image.clone(), @@ -714,7 +693,7 @@ impl<'a> Kubernetes<'a> { }; k8s_helpers::create_replica_set( - format!("{}-{}-{}", image.validator_type(), image.tag(), rpc_index), + format!("{}-{}-{}", image.node_type(), image.tag(), rpc_index), self.namespace.clone(), label_selector.clone(), image.clone(), @@ -756,22 +735,15 @@ impl<'a> Kubernetes<'a> { ..Default::default() }]); - let mut command = - vec!["/home/solana/k8s-cluster-scripts/client-startup-script.sh".to_string()]; - command.extend(self.generate_client_command_flags()); + let command = self.client_config.build_command()?; k8s_helpers::create_replica_set( - format!( - "{}-{}-{}", - image.validator_type(), - image.tag(), - client_index - ), + format!("{}-{}-{}", image.node_type(), image.tag(), client_index), self.namespace.clone(), label_selector.clone(), image.clone(), env_vars, - command.clone(), + command, accounts_volume, accounts_volume_mount, self.pod_requests.requests.clone(), diff --git a/src/lib.rs b/src/lib.rs index 61bdf0f..5a78ad9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -46,7 +46,15 @@ struct GenesisProgram<'a> { } #[derive(Debug, Clone, Copy, PartialEq, Display)] -pub enum ValidatorType { +pub enum ClientType { + #[strum(serialize = "bench-tps-client")] + BenchTps, + #[strum(serialize = "generic-client")] + Generic, +} + +#[derive(Debug, Clone, Copy, PartialEq, Display)] +pub enum NodeType { #[strum(serialize = "bootstrap-validator")] Bootstrap, #[strum(serialize = "validator")] @@ -54,16 +62,16 @@ pub enum ValidatorType { #[strum(serialize = "rpc-node")] RPC, #[strum(serialize = "client")] - Client(/* client index */ usize), + Client(ClientType, /* client index */ usize), } -impl ValidatorType { +impl NodeType { fn script(&self) -> &'static str { match self { - ValidatorType::Bootstrap => startup_scripts::StartupScripts::bootstrap(), - ValidatorType::Standard => startup_scripts::StartupScripts::validator(), - ValidatorType::RPC => startup_scripts::StartupScripts::rpc(), - ValidatorType::Client(_) => startup_scripts::StartupScripts::client(), + NodeType::Bootstrap => startup_scripts::StartupScripts::bootstrap(), + NodeType::Standard => startup_scripts::StartupScripts::validator(), + NodeType::RPC => startup_scripts::StartupScripts::rpc(), + NodeType::Client(_, _) => startup_scripts::StartupScripts::client(), } } } @@ -108,9 +116,9 @@ pub mod genesis; pub mod k8s_helpers; pub mod kubernetes; pub mod ledger_helper; +pub mod node; pub mod release; pub mod startup_scripts; -pub mod validator; pub mod validator_config; static BUILD: Emoji = Emoji("👷 ", ""); @@ -284,7 +292,7 @@ pub async fn fetch_spl(solana_root_path: &Path) -> Result<(), Box) -> Vec { +pub fn parse_and_format_transparent_args(bench_tps_args: Option<&str>) -> Vec { if let Some(args) = bench_tps_args { let mut val_args: Vec<_> = args .split_whitespace() @@ -314,3 +322,13 @@ pub fn check_directory(path: &Path, description: &str) -> Result<(), Box Result<(), String> { + let parts: Vec<&str> = image.split('/').collect(); + if parts.len() != 2 || !parts[1].contains(':') { + return Err( + "Invalid Docker image format. Expected /:".into(), + ); + } + Ok(()) +} diff --git a/src/main.rs b/src/main.rs index 186844a..63416ed 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,5 +1,5 @@ use { - clap::{command, value_t_or_exit, Arg, ArgGroup}, + clap::{command, value_t_or_exit, Arg, ArgGroup, SubCommand}, log::*, solana_clap_v3_utils::input_parsers::pubkey_of, solana_ledger::blockstore_cleanup_service::{ @@ -10,7 +10,7 @@ use { strum::VariantNames, validator_lab::{ check_directory, - client_config::ClientConfig, + client_config::{BenchTpsConfig, ClientConfig, GenericClientConfig}, cluster_images::ClusterImages, docker::{DockerConfig, DockerImage}, genesis::{ @@ -21,11 +21,12 @@ use { }, kubernetes::{Kubernetes, PodRequests}, ledger_helper::LedgerHelper, - parse_and_format_bench_tps_args, + node::{LabelType, Node}, + parse_and_format_transparent_args, release::{BuildConfig, BuildType, DeployMethod}, - validator::{LabelType, Validator}, + validate_docker_image, validator_config::ValidatorConfig, - ClusterDataRoot, EnvironmentConfig, Metrics, ValidatorType, SOLANA_RELEASE, + ClientType, ClusterDataRoot, EnvironmentConfig, Metrics, NodeType, SOLANA_RELEASE, }, }; @@ -185,7 +186,7 @@ fn parse_matches() -> clap::ArgMatches { .long("image-name") .takes_value(true) .default_value("k8s-image") - .help("Docker image name. Will be prepended with validator_type (bootstrap or validator)"), + .help("Docker image name. Will be prepended with node_type (bootstrap or validator)"), ) .arg( Arg::with_name("base_image") @@ -269,68 +270,131 @@ fn parse_matches() -> clap::ArgMatches { .help("Number of rpc nodes") ) // Client Config - .arg( - Arg::with_name("number_of_clients") - .long("num-clients") - .short('c') - .takes_value(true) - .default_value("0") - .help("Number of clients") - ) - .arg( - Arg::with_name("client_type") - .long("client-type") - .takes_value(true) - .default_value("tpu-client") - .possible_values(["tpu-client", "rpc-client"]) - .help("Client Config. Set Client Type"), - ) - .arg( - Arg::with_name("client_to_run") - .long("client-to-run") - .takes_value(true) - .default_value("bench-tps") - .possible_values(["bench-tps", "idle"]) - .help("Client Config. Set Client to run"), - ) - .arg( - Arg::with_name("bench_tps_args") - .long("bench-tps-args") - .value_name("KEY VALUE") - .takes_value(true) - .multiple(true) - .number_of_values(1) - .help("Client Config. - User can optionally provide extraArgs that are transparently - supplied to the client program as command line parameters. - For example, - --bench-tps-args 'tx-count=5000 thread-batch-sleep-ms=250' - This will start bench-tps clients, and supply '--tx-count 5000 --thread-batch-sleep-ms 250' - to the bench-tps client."), - ) - .arg( - Arg::with_name("client_target_node") - .long("client-target-node") - .takes_value(true) - .value_name("PUBKEY") - .help("Client Config. Optional: Specify an exact node to send transactions to - Not supported yet. TODO..."), - ) - .arg( - Arg::with_name("client_duration_seconds") - .long("client-duration-seconds") - .takes_value(true) - .default_value("7500") - .value_name("SECS") - .help("Client Config. Seconds to run benchmark, then exit"), + .subcommand(SubCommand::with_name("bench-tps") + .about("Run the bench-tps client") + .arg( + Arg::with_name("number_of_clients") + .long("num-clients") + .short('c') + .takes_value(true) + .default_value("1") + .help("Number of clients"), + ) + .arg( + Arg::with_name("client_duration_seconds") + .long("client-duration-seconds") + .takes_value(true) + .default_value("7500") + .value_name("SECS") + .help("Seconds to run benchmark, then exit"), + ) + .arg( + Arg::with_name("client_type") + .long("client-type") + .takes_value(true) + .default_value("tpu-client") + .possible_values(["tpu-client", "rpc-client"]) + .help("Set Client Type"), + ) + .arg( + Arg::with_name("client_to_run") + .long("client-to-run") + .takes_value(true) + .default_value("bench-tps") + .possible_values(["bench-tps", "idle"]) + .help("Client Config. Set Client to run"), + ) + .arg( + Arg::with_name("bench_tps_args") + .long("bench-tps-args") + .value_name("KEY VALUE") + .takes_value(true) + .multiple(true) + .number_of_values(1) + .help("Client Config. + User can optionally provide extraArgs that are transparently + supplied to the client program as command line parameters. + For example, + --bench-tps-args 'tx-count=5000 thread-batch-sleep-ms=250' + This will start bench-tps clients, and supply '--tx-count 5000 --thread-batch-sleep-ms 250' + to the bench-tps client."), + ) + .arg( + Arg::with_name("client_wait_for_n_nodes") + .long("client-wait-for-n-nodes") + .short('N') + .takes_value(true) + .value_name("NUM") + .help("Optional: Wait for NUM nodes to converge"), + ) + .arg( + Arg::with_name("client_target_node") + .long("client-target-node") + .takes_value(true) + .value_name("PUBKEY") + .help("Client Config. Optional: Specify an exact node to send transactions to + Not supported yet. TODO..."), + ) ) - .arg( - Arg::with_name("client_wait_for_n_nodes") - .long("client-wait-for-n-nodes") - .short('N') - .takes_value(true) - .value_name("NUM") - .help("Client Config. Optional: Wait for NUM nodes to converge"), + .subcommand(SubCommand::with_name("generic-client") + .about("Run a generic client") + .arg( + Arg::with_name("number_of_clients") + .long("num-clients") + .short('c') + .takes_value(true) + .default_value("1") + .help("Number of clients"), + ) + .arg( + Arg::with_name("client_duration_seconds") + .long("client-duration-seconds") + .takes_value(true) + .default_value("7500") + .value_name("SECS") + .help("Seconds to run benchmark, then exit"), + ) + .arg( + Arg::with_name("docker_image") + .long("docker-image") + .takes_value(true) + .value_name("/:") + .validator(validate_docker_image) + .required(true) + .help("Name of docker image to pull and run"), + ) + .arg( + Arg::with_name("executable_path") + .long("executable-path") + .takes_value(true) + .required(true) + .help("The path of the executable to run inside the container. e.g. /home/solana/"), + ) + .arg( + Arg::with_name("delay_start") + .long("delay-start") + .takes_value(true) + .required(true) + .default_value("0") + .help("Wait for `delay-start` seconds after all validators are deployed to deploy client. + Use case: If client needs to connect to a specific node, but that node hasn't fully deployed yet + the client may not be able to resolve the node's endpoint. `--delay-start` is used to wait so + validators can deploy fully before launching the client. Currently only used for generic clients + since similar functionality is built into bench-tps-client"), + ) + .arg( + Arg::with_name("generic_client_args") + .long("generic-client-args") + .takes_value(true) + .value_name("KEY VALUE") + .number_of_values(1) + .help("User can provide args that are transparently + supplied to the client program as command line parameters. + For example, + --generic-client-args 'thread-sleep-ms=0 spam-type=mixed' + This will start the generic clients, and supply '--thread-sleep-ms 0 --spam-type mixed' + to the generic client executable."), + ) ) // Heterogeneous Cluster Config .arg( @@ -409,20 +473,37 @@ async fn main() -> Result<(), Box> { let num_validators = value_t_or_exit!(matches, "number_of_validators", usize); let num_rpc_nodes = value_t_or_exit!(matches, "number_of_rpc_nodes", usize); - let client_config = ClientConfig { - num_clients: value_t_or_exit!(matches, "number_of_clients", usize), - client_type: matches.value_of("client_type").unwrap().to_string(), - client_to_run: matches.value_of("client_to_run").unwrap().to_string(), - bench_tps_args: parse_and_format_bench_tps_args(matches.value_of("bench_tps_args")), - client_target_node: pubkey_of(&matches, "client_target_node"), - client_duration_seconds: value_t_or_exit!(matches, "client_duration_seconds", u64), - client_wait_for_n_nodes: matches - .value_of("client_wait_for_n_nodes") - .map(|value_str| { - value_str - .parse() - .expect("Invalid value for client_wait_for_n_nodes") - }), + let client_config = if let Some(matches) = matches.subcommand_matches("bench-tps") { + let bench_tps_config = BenchTpsConfig { + num_clients: value_t_or_exit!(matches, "number_of_clients", usize), + client_duration_seconds: value_t_or_exit!(matches, "client_duration_seconds", u64), + client_type: matches.value_of("client_type").unwrap().to_string(), + client_to_run: matches.value_of("client_to_run").unwrap().to_string(), + bench_tps_args: parse_and_format_transparent_args(matches.value_of("bench_tps_args")), + client_wait_for_n_nodes: matches + .value_of("client_wait_for_n_nodes") + .map(|value_str| { + value_str + .parse() + .expect("Invalid value for client_wait_for_n_nodes") + }), + client_target_node: pubkey_of(matches, "client_target_node"), + }; + + ClientConfig::BenchTps(bench_tps_config) + } else if let Some(matches) = matches.subcommand_matches("generic-client") { + let generic_config = GenericClientConfig { + num_clients: value_t_or_exit!(matches, "number_of_clients", usize), + client_duration_seconds: value_t_or_exit!(matches, "client_duration_seconds", u64), + image: matches.value_of("docker_image").unwrap().to_string(), + args: parse_and_format_transparent_args(matches.value_of("generic_client_args")), + executable_path: value_t_or_exit!(matches, "executable_path", PathBuf), + delay_start: value_t_or_exit!(matches, "delay_start", u64), + }; + + ClientConfig::Generic(generic_config) + } else { + ClientConfig::None }; let deploy_method = if let Some(local_path) = matches.value_of("local_path") { @@ -598,7 +679,7 @@ async fn main() -> Result<(), Box> { genesis.generate_faucet()?; info!("Generated faucet account"); - genesis.generate_accounts(ValidatorType::Bootstrap, 1, None)?; + genesis.generate_accounts(NodeType::Bootstrap, 1, None)?; info!("Generated bootstrap account"); // creates genesis and writes to binary file @@ -609,10 +690,10 @@ async fn main() -> Result<(), Box> { } // generate standard validator accounts - genesis.generate_accounts(ValidatorType::Standard, num_validators, Some(&image_tag))?; + genesis.generate_accounts(NodeType::Standard, num_validators, Some(&image_tag))?; info!("Generated {num_validators} validator account(s)"); - genesis.generate_accounts(ValidatorType::RPC, num_rpc_nodes, Some(&image_tag))?; + genesis.generate_accounts(NodeType::RPC, num_rpc_nodes, Some(&image_tag))?; info!("Generated {num_rpc_nodes} rpc account(s)"); let ledger_dir = config_directory.join("bootstrap-validator"); @@ -628,62 +709,77 @@ async fn main() -> Result<(), Box> { let mut cluster_images = ClusterImages::default(); if deploy_bootstrap_validator { - let bootstrap_validator = Validator::new(DockerImage::new( + let bootstrap_validator = Node::new(DockerImage::new( registry_name.clone(), - ValidatorType::Bootstrap, + NodeType::Bootstrap, image_name.clone(), image_tag.clone(), )); - cluster_images.set_item(bootstrap_validator, ValidatorType::Bootstrap); + cluster_images.set_item(bootstrap_validator); } if num_validators > 0 { - let validator = Validator::new(DockerImage::new( + let validator = Node::new(DockerImage::new( registry_name.clone(), - ValidatorType::Standard, + NodeType::Standard, image_name.clone(), image_tag.clone(), )); - cluster_images.set_item(validator, ValidatorType::Standard); + cluster_images.set_item(validator); } if num_rpc_nodes > 0 { - let rpc_node = Validator::new(DockerImage::new( + let rpc_node = Node::new(DockerImage::new( registry_name.clone(), - ValidatorType::RPC, + NodeType::RPC, image_name.clone(), image_tag.clone(), )); - cluster_images.set_item(rpc_node, ValidatorType::RPC); + cluster_images.set_item(rpc_node); } - if client_config.num_clients > 0 { - genesis.create_client_accounts( - client_config.num_clients, - &client_config.bench_tps_args, - DEFAULT_CLIENT_LAMPORTS_PER_SIGNATURE, - &config_directory, - cluster_data_root.get_root_path(), - )?; - info!("Client accounts created"); + if let ClientConfig::BenchTps(ref bench_tps_config) = client_config { + if bench_tps_config.num_clients > 0 { + genesis.create_client_accounts( + bench_tps_config.num_clients, + &bench_tps_config.bench_tps_args, + DEFAULT_CLIENT_LAMPORTS_PER_SIGNATURE, + &config_directory, + cluster_data_root.get_root_path(), + )?; + info!("Client accounts created"); + } } - for client_index in 0..client_config.num_clients { - let client = Validator::new(DockerImage::new( - registry_name.clone(), - ValidatorType::Client(client_index), - image_name.clone(), - image_tag.clone(), - )); - cluster_images.set_item(client, ValidatorType::Client(client_index)); + for client_index in 0..client_config.num_clients() { + let client = match client_config { + ClientConfig::BenchTps(_) => Node::new(DockerImage::new( + registry_name.clone(), + NodeType::Client(ClientType::BenchTps, client_index), + image_name.clone(), + image_tag.clone(), + )), + ClientConfig::Generic(ref config) => { + Node::new(DockerImage::new_from_string(config.image.clone())?) + } + ClientConfig::None => unreachable!(), + }; + cluster_images.set_item(client); } for v in cluster_images.get_all() { + if let NodeType::Client(ClientType::Generic, _) = v.node_type() { + continue; + } docker.build_image(cluster_data_root.get_root_path(), v.image())?; - info!("Built {} image", v.validator_type()); + info!("Built {} image", v.node_type()); } - docker.push_images(cluster_images.get_all())?; + let images_to_push: Vec<&Node> = cluster_images + .get_all() + .filter(|v| !matches!(v.node_type(), NodeType::Client(ClientType::Generic, _))) + .collect(); + docker.push_images(images_to_push)?; info!("Pushed {} docker images", cluster_images.get_all().count()); // metrics secret create once and use by all pods @@ -726,7 +822,7 @@ async fn main() -> Result<(), Box> { ); bootstrap_validator.add_label( "validator/type", - bootstrap_validator.validator_type().to_string(), + bootstrap_validator.node_type().to_string(), LabelType::Info, ); bootstrap_validator.add_label( @@ -809,7 +905,7 @@ async fn main() -> Result<(), Box> { rpc_node.add_label( "rpc-node/type", - rpc_node.validator_type().to_string(), + rpc_node.node_type().to_string(), LabelType::Info, ); @@ -885,7 +981,7 @@ async fn main() -> Result<(), Box> { ); validator.add_label( "validator/type", - validator.validator_type().to_string(), + validator.node_type().to_string(), LabelType::Info, ); validator.add_label( @@ -917,40 +1013,51 @@ async fn main() -> Result<(), Box> { } } - for client in cluster_images.get_clients_mut() { - let client_index = if let ValidatorType::Client(index) = client.validator_type() { + if let ClientConfig::Generic(ref generic_config) = client_config { + info!( + "Waiting {}s before deploying client", + generic_config.delay_start + ); + std::thread::sleep(std::time::Duration::from_secs(generic_config.delay_start)); + } + + for client_node in cluster_images.get_clients_mut() { + let client_index = if let NodeType::Client(_, index) = client_node.node_type() { *index } else { return Err("Invalid Validator Type in Client".into()); }; let client_secret = kub_controller.create_client_secret(client_index, &config_directory)?; - client.set_secret(client_secret); + client_node.set_secret(client_secret); - kub_controller.deploy_secret(client.secret()).await?; + kub_controller.deploy_secret(client_node.secret()).await?; info!("Deployed Client {client_index} Secret"); - client.add_label( + client_node.add_label( "client/name", format!("client-{client_index}"), LabelType::Service, ); let client_replica_set = kub_controller.create_client_replica_set( - client.image(), - client.secret().metadata.name.clone(), - &client.all_labels(), + client_node.image(), + client_node.secret().metadata.name.clone(), + &client_node.all_labels(), client_index, )?; - client.set_replica_set(client_replica_set); + client_node.set_replica_set(client_replica_set); kub_controller - .deploy_replicas_set(client.replica_set()) + .deploy_replicas_set(client_node.replica_set()) .await?; info!("Deployed Client Replica Set ({client_index})"); - let client_service = - kub_controller.create_service("client-service", client_index, client.service_labels()); + let client_service = kub_controller.create_service( + "client-service", + client_index, + client_node.service_labels(), + ); kub_controller.deploy_service(&client_service).await?; info!("Deployed Client Service ({client_index})"); } diff --git a/src/validator.rs b/src/node.rs similarity index 89% rename from src/validator.rs rename to src/node.rs index 8f672f9..98d2589 100644 --- a/src/validator.rs +++ b/src/node.rs @@ -1,5 +1,5 @@ use { - crate::{docker::DockerImage, ValidatorType}, + crate::{docker::DockerImage, NodeType}, k8s_openapi::api::{apps::v1::ReplicaSet, core::v1::Secret}, std::{collections::BTreeMap, string::String}, }; @@ -10,8 +10,8 @@ pub enum LabelType { } #[derive(Clone)] -pub struct Validator { - validator_type: ValidatorType, +pub struct Node { + node_type: NodeType, image: DockerImage, secret: Secret, info_labels: BTreeMap, @@ -19,10 +19,10 @@ pub struct Validator { service_labels: BTreeMap, } -impl Validator { +impl Node { pub fn new(image: DockerImage) -> Self { Self { - validator_type: image.validator_type(), + node_type: image.node_type(), image, secret: Secret::default(), info_labels: BTreeMap::new(), @@ -39,8 +39,8 @@ impl Validator { &self.secret } - pub fn validator_type(&self) -> &ValidatorType { - &self.validator_type + pub fn node_type(&self) -> &NodeType { + &self.node_type } pub fn add_label(&mut self, key: K, value: V, label_type: LabelType)