Archive Count Metric

twoshark · Feb 16, 2023 · 9de5e55 · 9de5e55
2 parents efd1a88 + 3520f9a
commit 9de5e55
Show file tree

Hide file tree

Showing 7 changed files with 39 additions and 8 deletions.
diff --git a/dashboards/jsonnet/server_dashboard.jsonnet b/dashboards/jsonnet/server_dashboard.jsonnet
@@ -114,13 +114,18 @@ grafana.dashboard.new(
   grafana.graphPanel.new(
     'Upstreams Metrics',
     datasource='$datasource',
-    span=2,
+    span=3,
   )
   .addTarget(
     grafana.prometheus.target(
       'upstreams_healthy',
     )
   )
+  .addTarget(
+    grafana.prometheus.target(
+      'upstreams_archive',
+    )
+  )
   .addTarget(
     grafana.prometheus.target(
       'upstreams_configured',

diff --git a/src/metrics/metrics.go b/src/metrics/metrics.go
@@ -11,6 +11,7 @@ type MetricSet struct {
 	StartUpTime              *prometheus.HistogramVec
 	ConfiguredUpstreams      prometheus.Gauge
 	HealthyUpstreams         prometheus.Gauge
+	ArchiveUpstreams         prometheus.Gauge
 	MaxBlock                 prometheus.Gauge
 	EthSyncingLatency        *prometheus.HistogramVec
 	EthGetBlockNumberLatency *prometheus.HistogramVec
@@ -39,6 +40,12 @@ func Metrics() *MetricSet {
 					Help: "currently healthy upstream endpoints",
 				},
 			),
+			ArchiveUpstreams: promauto.NewGauge(
+				prometheus.GaugeOpts{
+					Name: "upstreams_archive",
+					Help: "count of healthy archive upstream endpoints",
+				},
+			),
 			MaxBlock: promauto.NewGauge(
 				prometheus.GaugeOpts{
 					Name: "max_ethereum_block",

diff --git a/src/server/balanceProxy.go b/src/server/balanceProxy.go
@@ -57,7 +57,8 @@ func (bp *BalanceProxy) LiveHandler(c echo.Context) error {
 // if there are any healthy upstreams, it will return a 200
 // otherwise it will return a 503 and disable the ingress
 func (bp *BalanceProxy) ReadyHandler(c echo.Context) error {
-	if bp.UpstreamManager.HealthyCount() > 0 {
+	counts := bp.UpstreamManager.ClientCounts()
+	if counts.Healthy > 0 {
 		return c.String(http.StatusOK, "ready")
 	}
 	return c.String(http.StatusServiceUnavailable, "no healthy upstreams")

diff --git a/src/server/server_test.go b/src/server/server_test.go
@@ -139,6 +139,7 @@ func (suite *ServerTestSuite) TestMetricHandler() {
 	assert.True(suite.T(), strings.Contains(response, "startup_time_ms"))
 	assert.True(suite.T(), strings.Contains(response, "upstreams_configured"))
 	assert.True(suite.T(), strings.Contains(response, "upstreams_healthy"))
+	assert.True(suite.T(), strings.Contains(response, "upstreams_archive"))
 	assert.True(suite.T(), strings.Contains(response, "latency_eth_syncing"))
 	assert.True(suite.T(), strings.Contains(response, "latency_eth_get_block_number"))
 	assert.True(suite.T(), strings.Contains(response, "latency_eth_get_balance"))

diff --git a/src/upstream/health.go b/src/upstream/health.go
@@ -25,19 +25,33 @@ func (m *Manager) StartHealthCheck() chan bool {
 				for i := range m.Clients {
 					m.Clients[i].EvaluatedHealthCheck()
 				}
-				metrics.Metrics().HealthyUpstreams.Set(float64(m.HealthyCount()))
+				clientCounts := m.ClientCounts()
+				metrics.Metrics().HealthyUpstreams.Set(float64(clientCounts.Healthy))
+				metrics.Metrics().ArchiveUpstreams.Set(float64(clientCounts.Archive))
 			}
 		}
 	}()
 	return quitHealthCheck
 }
 
-func (m *Manager) HealthyCount() int {
-	count := 0
+type ClientCounts struct {
+	Archive int
+	Healthy int
+}
+
+func (m *Manager) ClientCounts() ClientCounts {
+	healthyCount := 0
+	archiveCount := 0
 	for _, client := range m.Clients {
 		if client.Healthy() {
-			count++
+			healthyCount++
 		}
+		if client.IsArchive() {
+			archiveCount++
+		}
+	}
+	return ClientCounts{
+		Archive: archiveCount,
+		Healthy: healthyCount,
 	}
-	return count
 }
diff --git a/src/upstream/manager.go b/src/upstream/manager.go
@@ -61,7 +61,9 @@ func (m *Manager) ConnectAll() error {
 			break
 		}
 	}
-	metrics.Metrics().HealthyUpstreams.Set(float64(m.HealthyCount()))
+	clientCounts := m.ClientCounts()
+	metrics.Metrics().HealthyUpstreams.Set(float64(clientCounts.Healthy))
+	metrics.Metrics().ArchiveUpstreams.Set(float64(clientCounts.Archive))
 
 	if !anyAvailable {
 		return errNoUpstreamAvailable

diff --git a/src/upstream/manager_connect_test.go b/src/upstream/manager_connect_test.go
@@ -176,6 +176,7 @@ func (suite *ManagerTestSuite) verifyConnectAllTestCase(testCase connectAllTestC
 		// infer `Healthy()` from HeathCheck and Dial errors
 		healthy := testCase.clientExpects[i].HealthCheck.Err == nil && testCase.clientExpects[i].Dial.Err == nil
 		client.EXPECT().Healthy().Return(healthy).AnyTimes()
+		client.EXPECT().IsArchive().Return(false).AnyTimes()
 		client.EXPECT().SetHealth(healthy).AnyTimes()
 		mgr.Clients[i] = client
 	}