Integration of Atlantic Service and HTTP Client Infrastructure

.env.example

@@ -1,9 +1,8 @@
 ##### ORCHESTRATOR #####
 
-HOST=
-PORT=
-MAX_BLOCK_TO_PROCESS=
-SNOS_PROOF_LAYOUT=
+MADARA_ORCHESTRATOR_HOST=
+MADARA_ORCHESTRATOR_PORT=
+MADARA_ORCHESTRATOR_MAX_BLOCK_NO_TO_PROCESS=
 
 ##### AWS CONFIG #####
 
@@ -18,7 +17,7 @@ AWS_DEFAULT_REGION="localhost"
 ##### STORAGE #####
 
 DATA_STORAGE=
-AWS_S3_BUCKET_NAME=
+MADARA_ORCHESTRATOR_AWS_S3_BUCKET_NAME=
 
 ##### QUEUE #####
 
@@ -31,47 +30,48 @@ SQS_WORKER_TRIGGER_QUEUE_URL=
 ##### SNS #####
 
 ALERTS="sns"
-AWS_SNS_ARN="arn:aws:sns:us-east-1:000000000000:madara-orchestrator-arn"
+MADARA_ORCHESTRATOR_AWS_SNS_ARN="arn:aws:sns:us-east-1:000000000000:madara-orchestrator-arn"
 
 ##### DATABASE #####
 
 DATABASE=
-MONGODB_CONNECTION_STRING=
-DATABASE_NAME=
+MADARA_ORCHESTRATOR_MONGODB_CONNECTION_URL=
+MADARA_ORCHESTRATOR_DATABASE_NAME=
 
 ##### PROVER #####
 
 PROVER_SERVICE=
 
 ## if using sharp
-SHARP_CUSTOMER_ID=
-SHARP_URL=
-SHARP_USER_CRT=
-SHARP_USER_KEY=
-SHARP_SERVER_CRT=
+MADARA_ORCHESTRATOR_SHARP_CUSTOMER_ID=
+MADARA_ORCHESTRATOR_SHARP_URL=
+MADARA_ORCHESTRATOR_SHARP_USER_CRT=
+MADARA_ORCHESTRATOR_SHARP_USER_KEY=
+MADARA_ORCHESTRATOR_SHARP_SERVER_CRT=
+MADARA_ORCHESTRATOR_SHARP_PROOF_LAYOUT=
 
 
 ## if using atlantic
-ATLANTIC_API_KEY=
-ATLANTIC_URL=
-MOCK_FACT_HASH=
-PROVER_TYPE=
+MADARA_ORCHESTRATOR_ATLANTIC_API_KEY=
+MADARA_ORCHESTRATOR_ATLANTIC_URL=
+MADARA_ORCHESTRATOR_MOCK_FACT_HASH=
+MADARA_ORCHESTRATOR_PROVER_TYPE=
 
 ##### ON CHAIN CONFIG #####
 
 DA_LAYER=
 SETTLEMENT_LAYER=
-SETTLEMENT_RPC_URL=
-MADARA_RPC_URL=
-GPS_VERIFIER_CONTRACT_ADDRESS=
+MADARA_ORCHESTRATOR_ETHEREUM_SETTLEMENT_RPC_URL=
+MADARA_ORCHESTRATOR_MADARA_RPC_URL=
+MADARA_ORCHESTRATOR_GPS_VERIFIER_CONTRACT_ADDRESS=
 PRIVATE_KEY=
-ETHEREUM_PRIVATE_KEY=
-L1_CORE_CONTRACT_ADDRESS=
+MADARA_ORCHESTRATOR_ETHEREUM_PRIVATE_KEY=
+MADARA_ORCHESTRATOR_L1_CORE_CONTRACT_ADDRESS=
 
 ##### STARKNET SETTLEMENT (L3s) #####
-STARKNET_PRIVATE_KEY=
-STARKNET_ACCOUNT_ADDRESS=
+MADARA_ORCHESTRATOR_STARKNET_PRIVATE_KEY=
+MADARA_ORCHESTRATOR_STARKNET_ACCOUNT_ADDRESS=
 
 ##### Instrumentation #####
-OTEL_SERVICE_NAME=
-OTEL_COLLECTOR_ENDPOINT=
+MADARA_ORCHESTRATOR_OTEL_SERVICE_NAME=
+MADARA_ORCHESTRATOR_OTEL_COLLECTOR_ENDPOINT=

.github/workflows/coverage.yml

@@ -110,7 +110,7 @@ jobs:
           mv target/debug/madara ../madara-binary
           cd ..
           echo -e "
-          MADARA_BINARY_PATH=\"$(pwd)/madara-binary\"" >> .env.test
+          MADARA_ORCHESTRATOR_MADARA_BINARY_PATH=\"$(pwd)/madara-binary\"" >> .env.test
           cat .env.test
 
       - name: Getting neccesary files for testing
@@ -128,8 +128,8 @@ jobs:
 
       - name: Run llvm-cov tests
         env:
-          SETTLEMENT_RPC_URL: ${{ secrets.ETHEREUM_SEPOLIA_BLAST_RPC }}
-          RPC_FOR_SNOS: ${{ secrets.RPC_FOR_SNOS }}
+          MADARA_ORCHESTRATOR_ETHEREUM_SETTLEMENT_RPC_URL: ${{ secrets.ETHEREUM_SEPOLIA_BLAST_RPC }}
+          MADARA_ORCHESTRATOR_RPC_FOR_SNOS: ${{ secrets.RPC_FOR_SNOS }}
           # the self hosted runner has a different region so we override it here
           AWS_REGION: us-east-1
         run: RUST_LOG=debug RUST_BACKTRACE=1 cargo llvm-cov nextest --release --features testing --lcov --output-path lcov.info --test-threads=1 --workspace --exclude=e2e-tests --no-fail-fast

.github/workflows/e2e-test.yml

@@ -83,8 +83,8 @@ jobs:
 
       - name: Run e2e test
         env:
-          SETTLEMENT_RPC_URL: ${{ secrets.ETHEREUM_SEPOLIA_BLAST_RPC }}
-          RPC_FOR_SNOS: ${{ secrets.RPC_FOR_SNOS }}
+          MADARA_ORCHESTRATOR_ETHEREUM_SETTLEMENT_RPC_URL: ${{ secrets.ETHEREUM_SEPOLIA_BLAST_RPC }}
+          MADARA_ORCHESTRATOR_RPC_FOR_SNOS: ${{ secrets.RPC_FOR_SNOS }}
           # the self hosted runner has a different region so we override it here
           AWS_REGION: us-east-1
         run: |

CHANGELOG.md

@@ -7,12 +7,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 ## Added
 
 - tests: http_client tests added
+- Added Atlantic proving service integration
 - setup functions added for cloud and db
+- Added cli args support for all the services
+- Setup functions added for cloud and db
 - panic handling in process job
 - upgrade ETH L1 bridge for withdrawals to work
 - added makefile and submodules
 - Endpoints for triggering processing and verification jobs
-- Added Atlantic proving service integration
 - Add multiple queues for processing and verification based on job type
 - added logs
 - added MongoDB migrations using nodejs

Cargo.toml

@@ -87,6 +87,7 @@ lazy_static = "1.4.0"
 stark_evm_adapter = "0.1.1"
 hex = "0.4"
 itertools = "0.13.0"
+clap = { version = "4.4", features = ["derive", "env"] }
 mockall_double = "0.3.1"
 hyper = { version = "0.14", features = ["full"] }
 mockall = "0.13.0"

crates/da-clients/ethereum/src/lib.rs

@@ -13,19 +13,30 @@ use da_client_interface::{DaClient, DaVerificationStatus};
 use mockall::automock;
 use mockall::predicate::*;
 use reqwest::Client;
+use serde::{Deserialize, Serialize};
 use url::Url;
-use utils::settings::Settings;
 
-use crate::config::EthereumDaConfig;
-
-pub const DA_SETTINGS_NAME: &str = "ethereum";
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct EthereumDaValidatedArgs {
+    pub ethereum_da_rpc_url: Url,
+}
 
-pub mod config;
 pub struct EthereumDaClient {
     #[allow(dead_code)]
     provider: RootProvider<Ethereum, Http<Client>>,
 }
 
+impl EthereumDaClient {
+    pub async fn new_with_args(ethereum_da_params: &EthereumDaValidatedArgs) -> Self {
+        let client = RpcClient::new_http(
+            Url::from_str(ethereum_da_params.ethereum_da_rpc_url.as_str())
+                .expect("Failed to parse ethereum_da_rpc_url"),
+        );
+        let provider = ProviderBuilder::<_, Ethereum>::new().on_client(client);
+        Self { provider }
+    }
+}
+
 #[automock]
 #[async_trait]
 impl DaClient for EthereumDaClient {
@@ -47,14 +58,3 @@ impl DaClient for EthereumDaClient {
         131072
     }
 }
-
-impl EthereumDaClient {
-    pub fn new_with_settings(settings: &impl Settings) -> Self {
-        let config = EthereumDaConfig::new_with_settings(settings)
-            .expect("Not able to create EthereumDaClient from given settings.");
-        let client =
-            RpcClient::new_http(Url::from_str(config.rpc_url.as_str()).expect("Failed to parse SETTLEMENT_RPC_URL"));
-        let provider = ProviderBuilder::<_, Ethereum>::new().on_client(client);
-        EthereumDaClient { provider }
-    }
-}

crates/orchestrator/Cargo.toml

@@ -32,6 +32,7 @@ bytes.workspace = true
 c-kzg = { workspace = true }
 cairo-vm = { workspace = true }
 chrono = { workspace = true }
+clap = { workspace = true }
 color-eyre = { workspace = true }
 da-client-interface = { workspace = true }
 dotenvy = { workspace = true }
@@ -63,6 +64,7 @@ starknet = { workspace = true }
 starknet-core = { workspace = true }
 starknet-os = { workspace = true }
 starknet-settlement-client = { workspace = true }
+strum = { workspace = true }
 strum_macros = { workspace = true }
 tempfile = { workspace = true }
 thiserror = { workspace = true }

crates/orchestrator/src/alerts/aws_sns/mod.rs

@@ -1,28 +1,24 @@
-mod config;
-
-use std::sync::Arc;
-
 use async_trait::async_trait;
+use aws_config::SdkConfig;
 use aws_sdk_sns::Client;
-use utils::settings::Settings;
 
-use crate::alerts::aws_sns::config::AWSSNSConfig;
 use crate::alerts::Alerts;
-use crate::config::ProviderConfig;
 
-pub const AWS_SNS_SETTINGS_NAME: &str = "sns";
+#[derive(Debug, Clone)]
+pub struct AWSSNSValidatedArgs {
+    // TODO: convert to ARN type, and validate it
+    // NOTE: aws is using str to represent ARN : https://docs.aws.amazon.com/sdk-for-rust/latest/dg/rust_sns_code_examples.html
+    pub topic_arn: String,
+}
 
 pub struct AWSSNS {
     client: Client,
     topic_arn: String,
 }
 
 impl AWSSNS {
-    pub async fn new_with_settings(settings: &impl Settings, provider_config: Arc<ProviderConfig>) -> Self {
-        let sns_config =
-            AWSSNSConfig::new_with_settings(settings).expect("Not able to get Aws sns config from provided settings");
-        let config = provider_config.get_aws_client_or_panic();
-        Self { client: Client::new(config), topic_arn: sns_config.sns_arn }
+    pub async fn new_with_args(aws_sns_params: &AWSSNSValidatedArgs, aws_config: &SdkConfig) -> Self {
+        Self { client: Client::new(aws_config), topic_arn: aws_sns_params.topic_arn.clone() }
     }
 }
 
@@ -39,4 +35,8 @@ impl Alerts for AWSSNS {
         log::info!("SNS topic created. Topic ARN: {}", topic_arn);
         Ok(())
     }
+
+    async fn get_topic_name(&self) -> String {
+        self.topic_arn.split(":").last().expect("Failed to get last part of topic ARN").to_string()
+    }
 }

crates/orchestrator/src/alerts/mod.rs

@@ -1,6 +1,5 @@
 use async_trait::async_trait;
 use mockall::automock;
-use utils::settings::Settings;
 
 pub mod aws_sns;
 
@@ -9,10 +8,11 @@ pub mod aws_sns;
 pub trait Alerts: Send + Sync {
     /// To send an alert message to our alert service
     async fn send_alert_message(&self, message_body: String) -> color_eyre::Result<()>;
+    async fn get_topic_name(&self) -> String;
     async fn create_alert(&self, topic_name: &str) -> color_eyre::Result<()>;
-    async fn setup(&self, settings_provider: Box<dyn Settings>) -> color_eyre::Result<()> {
-        let sns_topic_name = settings_provider.get_settings_or_panic("ALERT_TOPIC_NAME");
-        self.create_alert(&sns_topic_name).await?;
+    async fn setup(&self) -> color_eyre::Result<()> {
+        let topic_name = self.get_topic_name().await;
+        self.create_alert(&topic_name).await?;
         Ok(())
     }
 }

crates/orchestrator/src/cli/alert/aws_sns.rs

@@ -0,0 +1,14 @@
+use clap::Args;
+
+/// Parameters used to config AWS SNS.
+#[derive(Debug, Clone, Args)]
+#[group()]
+pub struct AWSSNSCliArgs {
+    /// Use the AWS SNS client
+    #[arg(long)]
+    pub aws_sns: bool,
+
+    /// The ARN of the SNS topic.
+    #[arg(env = "MADARA_ORCHESTRATOR_AWS_SNS_ARN", long, default_value = Some("arn:aws:sns:us-east-1:000000000000:madara-orchestrator-arn"))]
+    pub sns_arn: Option<String>,
+}

crates/orchestrator/src/cli/alert/mod.rs

@@ -0,0 +1,8 @@
+use crate::alerts::aws_sns::AWSSNSValidatedArgs;
+
+pub mod aws_sns;
+
+#[derive(Clone, Debug)]
+pub enum AlertValidatedArgs {
+    AWSSNS(AWSSNSValidatedArgs),
+}