graph, store: Add time-based connection validation to skip health checks for active connections

Connections that were used within the last 30 seconds (configurable via
GRAPH_STORE_CONNECTION_VALIDATION_IDLE_SECS) now skip the SELECT 67 health
check during pool recycle. This reduces connection checkout latency from
~4ms to ~0ms for frequently-used connections while still validating idle
connections to detect stale database connections.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

Author: lutter

graph/src/env/store.rs

@@ -166,6 +166,11 @@ pub struct EnvVarsStore {
     /// Set by `GRAPH_STORE_DISABLE_CHAIN_HEAD_PTR_CACHE`. Default is false.
     /// Set to true to disable chain_head_ptr caching (safety escape hatch).
     pub disable_chain_head_ptr_cache: bool,
+    /// Minimum idle time before running connection health check (SELECT 67).
+    /// Connections used more recently than this threshold skip validation.
+    /// Set to 0 to always validate (previous behavior).
+    /// Set by `GRAPH_STORE_CONNECTION_VALIDATION_IDLE_SECS`. Default is 30 seconds.
+    pub connection_validation_idle_secs: Duration,
 }
 
 // This does not print any values avoid accidentally leaking any sensitive env vars
@@ -228,6 +233,7 @@ impl TryFrom<InnerStore> for EnvVarsStore {
             account_like_max_unique_ratio: x.account_like_max_unique_ratio.map(|r| r.0),
             disable_call_cache: x.disable_call_cache,
             disable_chain_head_ptr_cache: x.disable_chain_head_ptr_cache,
+            connection_validation_idle_secs: Duration::from_secs(x.connection_validation_idle_secs),
         };
         if let Some(timeout) = vars.batch_timeout {
             if timeout < 2 * vars.batch_target_duration {
@@ -337,6 +343,8 @@ pub struct InnerStore {
     disable_call_cache: bool,
     #[envconfig(from = "GRAPH_STORE_DISABLE_CHAIN_HEAD_PTR_CACHE", default = "false")]
     disable_chain_head_ptr_cache: bool,
+    #[envconfig(from = "GRAPH_STORE_CONNECTION_VALIDATION_IDLE_SECS", default = "30")]
+    connection_validation_idle_secs: u64,
 }
 
 #[derive(Clone, Copy, Debug)]

store/postgres/src/pool/manager.rs

@@ -35,6 +35,8 @@ pub struct ConnectionManager {
     connection_url: String,
     state_tracker: StateTracker,
     error_counter: Counter,
+    /// Connections idle for less than this threshold skip the SELECT 67 health check
+    validation_idle_threshold: Duration,
 }
 
 impl ConnectionManager {
@@ -44,6 +46,7 @@ impl ConnectionManager {
         state_tracker: StateTracker,
         registry: &MetricsRegistry,
         const_labels: HashMap<String, String>,
+        validation_idle_threshold: Duration,
     ) -> Self {
         let error_counter = registry
             .global_counter(
@@ -58,6 +61,7 @@ impl ConnectionManager {
             connection_url,
             state_tracker,
             error_counter,
+            validation_idle_threshold,
         }
     }
 
@@ -105,11 +109,20 @@ impl deadpool::managed::Manager for ConnectionManager {
     async fn recycle(
         &self,
         obj: &mut Self::Type,
-        _metrics: &deadpool::managed::Metrics,
+        metrics: &deadpool::managed::Metrics,
     ) -> RecycleResult<Self::Error> {
         if std::thread::panicking() || obj.is_broken() {
             return Err(RecycleError::Message("Broken connection".into()));
         }
+
+        // Skip health check if connection was used recently
+        if self.validation_idle_threshold > Duration::ZERO
+            && metrics.last_used() < self.validation_idle_threshold
+        {
+            return Ok(());
+        }
+
+        // Run SELECT 67 only for idle connections
         let res = diesel::select(67_i32.into_sql::<diesel::sql_types::Integer>())
             .execute(obj)
             .await

store/postgres/src/pool/mod.rs

@@ -476,6 +476,7 @@ impl PoolInner {
             state_tracker.clone(),
             &registry,
             const_labels.clone(),
+            ENV_VARS.store.connection_validation_idle_secs,
         );
 
         let timeouts = Timeouts {