feat(telemetry): Add Prometheus metrics endpoint for node monitoring

Cargo.toml

@@ -179,6 +179,12 @@ txtx-core = { version = "0.4.15" }
 txtx-gql = { version = "0.3.9" }
 txtx-supervisor-ui = { version = "0.2.10", default-features = false }
 
+opentelemetry = { version = "0.28", default-features = false, features = ["metrics"] }
+opentelemetry_sdk = { version = "0.28", default-features = false, features = ["rt-tokio", "metrics"] }
+opentelemetry-prometheus = { version = "0.28", default-features = false }
+prometheus = { version = "0.13", default-features = false }
+axum = { version = "0.8", default-features = false, features = ["tokio", "http1"] }
+
 # [patch.crates-io]
 ## Local
 # txtx-addon-kit = { path = "../txtx/crates/txtx-addon-kit" }

crates/cli/Cargo.toml

@@ -79,6 +79,7 @@ postgres = ["surfpool-gql/postgres", "surfpool-core/postgres"]
 version_check = []
 subgraph = ["surfpool-core/subgraph"]
 register-tracing = ["surfpool-core/register-tracing"]
+prometheus = ["surfpool-core/prometheus"]
 
 [target.'cfg(not(target_os = "windows"))'.dependencies]
 fork = "0.2.0"

crates/cli/src/cli/mod.rs

@@ -264,6 +264,20 @@ pub struct StartSimnet {
     /// When multiple files are provided, later files override earlier ones for duplicate keys.
     #[arg(long = "snapshot")]
     pub snapshot: Vec<String>,
+    /// Enable Prometheus metrics endpoint
+    #[cfg(feature = "prometheus")]
+    /// Enable Prometheus metrics endpoint
+    #[arg(long = "metrics-enabled", env = "SURFPOOL_METRICS_ENABLED")]
+    pub metrics_enabled: bool,
+
+    #[cfg(feature = "prometheus")]
+    /// Prometheus metrics endpoint address
+    #[arg(
+        long = "metrics-addr",
+        default_value = "0.0.0.0:9000",
+        env = "SURFPOOL_METRICS_ADDR"
+    )]
+    pub metrics_addr: String,
     /// Skip signature verification for all transactions (eg. surfpool start --skip-signature-verification)
     #[clap(long = "skip-signature-verification", action=ArgAction::SetTrue, default_value = "false")]
     pub skip_signature_verification: bool,

crates/cli/src/cli/simnet/mod.rs

@@ -63,6 +63,27 @@ pub async fn handle_start_local_surfnet_command(
     let (mut surfnet_svm, simnet_events_rx, geyser_events_rx) =
         SurfnetSvm::new_with_db(cmd.db.as_deref(), &cmd.surfnet_id)
             .map_err(|e| format!("Failed to initialize Surfnet SVM: {}", e))?;
+    #[cfg(feature = "prometheus")]
+    {
+        if cmd.metrics_enabled {
+            let handle = tokio::runtime::Handle::current();
+            match surfpool_core::telemetry::init_from_config(&cmd.metrics_addr, &handle) {
+                Err(e) => {
+                    let _ = surfnet_svm
+                        .simnet_events_tx
+                        .send(SimnetEvent::warn(format!("Metrics init failed: {}", e)));
+                }
+                Ok(_) => {
+                    use surfpool_types::DEFAULT_NETWORK_HOST;
+
+                    let _ = surfnet_svm.simnet_events_tx.send(SimnetEvent::info(format!(
+                        "Metrics available at http://{}/metrics",
+                        DEFAULT_NETWORK_HOST
+                    )));
+                }
+            }
+        }
+    }
 
     // Apply feature configuration from CLI flags
     let feature_config = cmd.feature_config();
@@ -321,6 +342,9 @@ async fn start_service(
         let _ = explorer_handle.stop(true).await;
     }
 
+    #[cfg(feature = "prometheus")]
+    surfpool_core::telemetry::shutdown();
+
     Ok(())
 }
 

crates/core/Cargo.toml

@@ -100,7 +100,12 @@ anchor-lang-idl = { workspace = true }
 txtx-addon-kit = { workspace = true }
 txtx-addon-network-svm-types = { workspace = true }
 txtx-addon-network-svm = { workspace = true }
-
+# Prometheus metrics - declare normally, control via feature
+opentelemetry = { version = "0.28", default-features = false, features = ["metrics"], optional = true }
+opentelemetry_sdk = { version = "0.28", default-features = false, features = ["rt-tokio", "metrics"], optional = true }
+opentelemetry-prometheus = { version = "0.28", default-features = false, optional = true }
+prometheus = { version = "0.13", default-features = false, optional = true }
+axum = { version = "0.8", default-features = false, features = ["tokio", "http1"], optional = true }
 
 [dev-dependencies]
 test-case = { workspace = true }
@@ -116,3 +121,4 @@ ignore_tests_ci = []
 geyser_plugin = []  # Disabled: solana-geyser-plugin-manager conflicts with litesvm 0.9.1
 subgraph = ["surfpool-subgraph"]
 register-tracing = ["litesvm/register-tracing"]
+prometheus = ["dep:opentelemetry", "dep:opentelemetry_sdk", "dep:opentelemetry-prometheus", "dep:prometheus", "dep:axum"]

crates/core/src/lib.rs

@@ -17,6 +17,7 @@ pub mod runloops;
 pub mod scenarios;
 pub mod storage;
 pub mod surfnet;
+pub mod telemetry;
 pub mod types;
 
 use crossbeam_channel::{Receiver, Sender};

crates/core/src/rpc/admin.rs

@@ -3,7 +3,7 @@ use std::time::Duration;
 use jsonrpc_core::{BoxFuture, Result};
 use jsonrpc_derive::rpc;
 use solana_client::rpc_custom_error::RpcCustomError;
-use surfpool_types::{SimnetCommand, SimnetEvent};
+use surfpool_types::{SimnetCommand, SimnetEvent, SurfpoolStatus};
 use txtx_addon_network_svm_types::subgraph::PluginConfig;
 use uuid::Uuid;
 
@@ -192,6 +192,9 @@ pub trait AdminRpc {
     /// - This method is useful for monitoring system uptime and verifying system health.
     #[rpc(meta, name = "startTime")]
     fn start_time(&self, meta: Self::Metadata) -> Result<String>;
+
+    #[rpc(meta, name = "surfpoolStatus")]
+    fn surfpool_status(&self, meta: Self::Metadata) -> Result<SurfpoolStatus>;
 }
 
 pub struct SurfpoolAdminRpc;
@@ -363,4 +366,17 @@ impl AdminRpc for SurfpoolAdminRpc {
         let datetime_utc: chrono::DateTime<chrono::Utc> = system_time.into();
         Ok(datetime_utc.to_rfc3339())
     }
+
+    fn surfpool_status(&self, meta: Self::Metadata) -> Result<SurfpoolStatus> {
+        // Ensure we have RunloopContext metadata
+        let Some(ctx) = meta else {
+            return Err(RpcCustomError::NodeUnhealthy {
+                num_slots_behind: None,
+            }
+            .into());
+        };
+
+        let status = ctx.svm_locker.with_svm_reader(|svm| svm.snapshot_status());
+        Ok(status)
+    }
 }

crates/core/src/runloops/mod.rs

@@ -449,6 +449,12 @@ pub async fn start_block_production_runloop(
                 svm_locker
                     .confirm_current_block(&remote_client_with_commitment)
                     .await?;
+
+                #[cfg(feature = "prometheus")]
+                {
+                    let snapshot = svm_locker.with_svm_reader(|svm| svm.snapshot_status());
+                    crate::telemetry::try_record_snapshot(&snapshot);
+                }
             }
         }
     }

crates/core/src/surfnet/locker.rs

@@ -1082,9 +1082,23 @@ impl SurfnetSvmLocker {
             }
         };
 
-        self.with_svm_writer(|svm_writer| {
-            svm_writer.write_executed_profile_result(signature, profile_result)
-        })?;
+        #[cfg(feature = "prometheus")]
+        let write_result = {
+            let (write_result, snapshot) = self.with_svm_writer(|svm| {
+                let res = svm.write_executed_profile_result(signature, profile_result);
+                let snap = svm.snapshot_status();
+                (res, snap)
+            });
+            crate::telemetry::try_record_snapshot(&snapshot);
+            write_result
+        };
+
+        #[cfg(not(feature = "prometheus"))]
+        let write_result = self
+            .with_svm_writer(|svm| svm.write_executed_profile_result(signature, profile_result));
+
+        write_result?;
+
         Ok(())
     }
 

crates/core/src/surfnet/svm.rs

@@ -52,8 +52,9 @@ use surfpool_types::{
     AccountChange, AccountProfileState, AccountSnapshot, DEFAULT_PROFILING_MAP_CAPACITY,
     DEFAULT_SLOT_TIME_MS, ExportSnapshotConfig, ExportSnapshotScope, FifoMap, Idl,
     OverrideInstance, ProfileResult, RpcProfileDepth, RpcProfileResultConfig,
-    RunbookExecutionStatusReport, SimnetEvent, SvmFeatureConfig, TransactionConfirmationStatus,
-    TransactionStatusEvent, UiAccountChange, UiAccountProfileState, UiProfileResult, VersionedIdl,
+    RunbookExecutionStatusReport, SimnetEvent, SurfpoolStatus, SvmFeatureConfig,
+    TransactionConfirmationStatus, TransactionStatusEvent, UiAccountChange, UiAccountProfileState,
+    UiProfileResult, VersionedIdl, WsSubscriptions,
     types::{
         ComputeUnitsEstimationResult, KeyedProfileResult, UiKeyedProfileResult, UuidOrSignature,
     },
@@ -330,6 +331,27 @@ impl SurfnetSvm {
         self.account_associated_data.shutdown();
     }
 
+    /// Returns a snapshot of the current SVM status for RPC and metrics.
+    pub fn snapshot_status(&self) -> SurfpoolStatus {
+        SurfpoolStatus {
+            slot: self.latest_epoch_info.absolute_slot,
+            epoch: self.latest_epoch_info.epoch,
+            slot_index: self.latest_epoch_info.slot_index,
+            transactions_count: self.transactions.count().unwrap_or(0),
+            transactions_processed: self.transactions_processed,
+            uptime_ms: std::time::SystemTime::now()
+                .duration_since(self.start_time)
+                .map(|d| d.as_millis() as u64)
+                .unwrap_or(0),
+            ws_subscriptions: WsSubscriptions {
+                signatures: self.signature_subscriptions.len(),
+                accounts: self.account_subscriptions.len(),
+                slots: self.slot_subscriptions.len(),
+                logs: self.logs_subscriptions.len(),
+            },
+        }
+    }
+
     /// Creates a clone of the SVM with overlay storage wrappers for all database-backed fields.
     /// This allows profiling transactions without affecting the underlying database.
     /// All storage writes are buffered in memory and discarded when the clone is dropped.