From dfe781d617f50230eb055e846fbc0dfd46d059ac Mon Sep 17 00:00:00 2001 From: Aditya Saha Date: Wed, 23 Oct 2024 13:53:28 -0400 Subject: [PATCH] Add load balancer monitoring endpoints --- monitoring.go | 196 ++++++++++++++++++++++++++++++++++++++++++- monitoring_test.go | 204 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 397 insertions(+), 3 deletions(-) diff --git a/monitoring.go b/monitoring.go index 937bb8d9..00feb256 100644 --- a/monitoring.go +++ b/monitoring.go @@ -10,9 +10,10 @@ import ( ) const ( - monitoringBasePath = "v2/monitoring" - alertPolicyBasePath = monitoringBasePath + "/alerts" - dropletMetricsBasePath = monitoringBasePath + "/metrics/droplet" + monitoringBasePath = "v2/monitoring" + alertPolicyBasePath = monitoringBasePath + "/alerts" + dropletMetricsBasePath = monitoringBasePath + "/metrics/droplet" + loadBalancerMetricsBasePath = monitoringBasePath + "/metrics/load_balancer" DropletCPUUtilizationPercent = "v1/insights/droplet/cpu" DropletMemoryUtilizationPercent = "v1/insights/droplet/memory_utilization_percent" @@ -67,6 +68,34 @@ type MonitoringService interface { GetDropletCachedMemory(context.Context, *DropletMetricsRequest) (*MetricsResponse, *Response, error) GetDropletFreeMemory(context.Context, *DropletMetricsRequest) (*MetricsResponse, *Response, error) GetDropletTotalMemory(context.Context, *DropletMetricsRequest) (*MetricsResponse, *Response, error) + + GetLoadBalancerFrontendHttpRequestsPerSecond(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) + GetLoadBalancerFrontendConnectionsCurrent(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) + GetLoadBalancerFrontendConnectionsLimit(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) + GetLoadBalancerFrontendCpuUtilization(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) + GetLoadBalancerFrontendNetworkThroughputHttp(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) + GetLoadBalancerFrontendNetworkThroughputUdp(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) + GetLoadBalancerFrontendNetworkThroughputTcp(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) + GetLoadBalancerFrontendNlbTcpNetworkThroughput(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) + GetLoadBalancerFrontendNlbUdpNetworkThroughput(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) + GetLoadBalancerFrontendFirewallDroppedBytes(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) + GetLoadBalancerFrontendFirewallDroppedPackets(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) + GetLoadBalancerFrontendHttpResponses(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) + GetLoadBalancerFrontendTlsConnectionsCurrent(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) + GetLoadBalancerFrontendTlsConnectionsLimit(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) + GetLoadBalancerFrontendTlsConnectionsExceedingRateLimit(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) + GetLoadBalancerDropletsHttpSessionDurationAvg(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) + GetLoadBalancerDropletsHttpSessionDuration50P(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) + GetLoadBalancerDropletsHttpSessionDuration95P(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) + GetLoadBalancerDropletsHttpResponseTimeAvg(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) + GetLoadBalancerDropletsHttpResponseTime50P(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) + GetLoadBalancerDropletsHttpResponseTime95P(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) + GetLoadBalancerDropletsHttpResponseTime99P(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) + GetLoadBalancerDropletsQueueSize(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) + GetLoadBalancerDropletsHttpResponses(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) + GetLoadBalancerDropletsConnections(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) + GetLoadBalancerDropletsHealthChecks(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) + GetLoadBalancerDropletsDowntime(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) } // MonitoringServiceOp handles communication with monitoring related methods of the @@ -163,6 +192,13 @@ type DropletBandwidthMetricsRequest struct { Direction string } +// LoadBalancerMetricsRequest holds the information needed to retrieve Load Balancer various metrics. +type LoadBalancerMetricsRequest struct { + LoadBalancerID string + Start time.Time + End time.Time +} + // MetricsResponse holds a Metrics query response. type MetricsResponse struct { Status string `json:"status"` @@ -372,3 +408,157 @@ func (s *MonitoringServiceOp) getDropletMetrics(ctx context.Context, path string return root, resp, err } + +// GetLoadBalancerFrontendHttpRequestsPerSecond retrieves frontend HTTP requests per second for a given load balancer. +func (s *MonitoringServiceOp) GetLoadBalancerFrontendHttpRequestsPerSecond(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) { + return s.getLoadBalancerMetrics(ctx, "/frontend_http_requests_per_second", args) +} + +// GetLoadBalancerFrontendConnectionsCurrent retrieves frontend total current active connections for a given load balancer. +func (s *MonitoringServiceOp) GetLoadBalancerFrontendConnectionsCurrent(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) { + return s.getLoadBalancerMetrics(ctx, "/frontend_connections_current", args) +} + +// GetLoadBalancerFrontendConnectionsLimit retrieves frontend max connections limit for a given load balancer. +func (s *MonitoringServiceOp) GetLoadBalancerFrontendConnectionsLimit(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) { + return s.getLoadBalancerMetrics(ctx, "/frontend_connections_limit", args) +} + +// GetLoadBalancerFrontendCpuUtilization retrieves frontend average percentage cpu utilization for a given load balancer. +func (s *MonitoringServiceOp) GetLoadBalancerFrontendCpuUtilization(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) { + return s.getLoadBalancerMetrics(ctx, "/frontend_cpu_utilization", args) +} + +// GetLoadBalancerFrontendNetworkThroughputHttp retrieves frontend HTTP throughput for a given load balancer. +func (s *MonitoringServiceOp) GetLoadBalancerFrontendNetworkThroughputHttp(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) { + return s.getLoadBalancerMetrics(ctx, "/frontend_network_throughput_http", args) +} + +// GetLoadBalancerFrontendNetworkThroughputUdp retrieves frontend UDP throughput for a given load balancer. +func (s *MonitoringServiceOp) GetLoadBalancerFrontendNetworkThroughputUdp(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) { + return s.getLoadBalancerMetrics(ctx, "/frontend_network_throughput_udp", args) +} + +// GetLoadBalancerFrontendNetworkThroughputTcp retrieves frontend TCP throughput for a given load balancer. +func (s *MonitoringServiceOp) GetLoadBalancerFrontendNetworkThroughputTcp(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) { + return s.getLoadBalancerMetrics(ctx, "/frontend_network_throughput_tcp", args) +} + +// GetLoadBalancerFrontendNlbTcpNetworkThroughput retrieves frontend TCP throughput for a given network load balancer. +func (s *MonitoringServiceOp) GetLoadBalancerFrontendNlbTcpNetworkThroughput(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) { + return s.getLoadBalancerMetrics(ctx, "/frontend_nlb_tcp_network_throughput", args) +} + +// GetLoadBalancerFrontendNlbUdpNetworkThroughput retrieves frontend UDP throughput for a given network load balancer. +func (s *MonitoringServiceOp) GetLoadBalancerFrontendNlbUdpNetworkThroughput(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) { + return s.getLoadBalancerMetrics(ctx, "/frontend_nlb_udp_network_throughput", args) +} + +// GetLoadBalancerFrontendFirewallDroppedBytes retrieves firewall dropped bytes for a given load balancer. +func (s *MonitoringServiceOp) GetLoadBalancerFrontendFirewallDroppedBytes(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) { + return s.getLoadBalancerMetrics(ctx, "/frontend_firewall_dropped_bytes", args) +} + +// GetLoadBalancerFrontendFirewallDroppedPackets retrieves firewall dropped packets for a given load balancer. +func (s *MonitoringServiceOp) GetLoadBalancerFrontendFirewallDroppedPackets(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) { + return s.getLoadBalancerMetrics(ctx, "/frontend_firewall_dropped_packets", args) +} + +// GetLoadBalancerFrontendHttpResponses retrieves frontend HTTP rate of response code for a given load balancer. +func (s *MonitoringServiceOp) GetLoadBalancerFrontendHttpResponses(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) { + return s.getLoadBalancerMetrics(ctx, "/frontend_http_responses", args) +} + +// GetLoadBalancerFrontendTlsConnectionsCurrent retrieves frontend current TLS connections rate for a given load balancer. +func (s *MonitoringServiceOp) GetLoadBalancerFrontendTlsConnectionsCurrent(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) { + return s.getLoadBalancerMetrics(ctx, "/frontend_tls_connections_current", args) +} + +// GetLoadBalancerFrontendTlsConnectionsLimit retrieves frontend max TLS connections limit for a given load balancer. +func (s *MonitoringServiceOp) GetLoadBalancerFrontendTlsConnectionsLimit(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) { + return s.getLoadBalancerMetrics(ctx, "/frontend_tls_connections_limit", args) +} + +// GetLoadBalancerFrontendTlsConnectionsExceedingRateLimit retrieves frontend closed TLS connections for exceeded rate limit for a given load balancer. +func (s *MonitoringServiceOp) GetLoadBalancerFrontendTlsConnectionsExceedingRateLimit(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) { + return s.getLoadBalancerMetrics(ctx, "/frontend_tls_connections_exceeding_rate_limit", args) +} + +// GetLoadBalancerDropletsHttpSessionDurationAvg retrieves droplet average HTTP session duration for a given load balancer. +func (s *MonitoringServiceOp) GetLoadBalancerDropletsHttpSessionDurationAvg(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) { + return s.getLoadBalancerMetrics(ctx, "/droplets_http_session_duration_avg", args) +} + +// GetLoadBalancerDropletsHttpSessionDuration50P retrieves droplet 50th percentile HTTP session duration for a given load balancer. +func (s *MonitoringServiceOp) GetLoadBalancerDropletsHttpSessionDuration50P(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) { + return s.getLoadBalancerMetrics(ctx, "/droplets_http_session_duration_50p", args) +} + +// GetLoadBalancerDropletsHttpSessionDuration95P retrieves droplet 95th percentile HTTP session duration for a given load balancer. +func (s *MonitoringServiceOp) GetLoadBalancerDropletsHttpSessionDuration95P(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) { + return s.getLoadBalancerMetrics(ctx, "/droplets_http_session_duration_95p", args) +} + +// GetLoadBalancerDropletsHttpResponseTimeAvg retrieves droplet average HTTP response time for a given load balancer. +func (s *MonitoringServiceOp) GetLoadBalancerDropletsHttpResponseTimeAvg(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) { + return s.getLoadBalancerMetrics(ctx, "/droplets_http_response_time_avg", args) +} + +// GetLoadBalancerDropletsHttpResponseTime50P retrieves droplet 50th percentile HTTP response time for a given load balancer. +func (s *MonitoringServiceOp) GetLoadBalancerDropletsHttpResponseTime50P(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) { + return s.getLoadBalancerMetrics(ctx, "/droplets_http_response_time_50p", args) +} + +// GetLoadBalancerDropletsHttpResponseTime95P retrieves droplet 95th percentile HTTP response time for a given load balancer. +func (s *MonitoringServiceOp) GetLoadBalancerDropletsHttpResponseTime95P(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) { + return s.getLoadBalancerMetrics(ctx, "/droplets_http_response_time_95p", args) +} + +// GetLoadBalancerDropletsHttpResponseTime99P retrieves droplet 99th percentile HTTP response time for a given load balancer. +func (s *MonitoringServiceOp) GetLoadBalancerDropletsHttpResponseTime99P(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) { + return s.getLoadBalancerMetrics(ctx, "/droplets_http_response_time_99p", args) +} + +// GetLoadBalancerDropletsQueueSize retrieves droplet queue size for a given load balancer. +func (s *MonitoringServiceOp) GetLoadBalancerDropletsQueueSize(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) { + return s.getLoadBalancerMetrics(ctx, "/droplets_queue_size", args) +} + +// GetLoadBalancerDropletsHttpResponses retrieves droplet HTTP rate of response code for a given load balancer. +func (s *MonitoringServiceOp) GetLoadBalancerDropletsHttpResponses(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) { + return s.getLoadBalancerMetrics(ctx, "/droplets_http_responses", args) +} + +// GetLoadBalancerDropletsConnections retrieves droplet active connections for a given load balancer. +func (s *MonitoringServiceOp) GetLoadBalancerDropletsConnections(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) { + return s.getLoadBalancerMetrics(ctx, "/droplets_connections", args) +} + +// GetLoadBalancerDropletsHealthChecks retrieves droplet health check status for a given load balancer. +func (s *MonitoringServiceOp) GetLoadBalancerDropletsHealthChecks(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) { + return s.getLoadBalancerMetrics(ctx, "/droplets_health_checks", args) +} + +// GetLoadBalancerDropletsDowntime retrieves droplet downtime status for a given load balancer. +func (s *MonitoringServiceOp) GetLoadBalancerDropletsDowntime(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) { + return s.getLoadBalancerMetrics(ctx, "/droplets_downtime", args) +} + +func (s *MonitoringServiceOp) getLoadBalancerMetrics(ctx context.Context, path string, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) { + fullPath := loadBalancerMetricsBasePath + path + req, err := s.client.NewRequest(ctx, http.MethodGet, fullPath, nil) + if err != nil { + return nil, nil, err + } + + q := req.URL.Query() + q.Add("lb_id", args.LoadBalancerID) + q.Add("start", fmt.Sprintf("%d", args.Start.Unix())) + q.Add("end", fmt.Sprintf("%d", args.End.Unix())) + req.URL.RawQuery = q.Encode() + + root := new(MetricsResponse) + resp, err := s.client.Do(ctx, req, root) + + return root, resp, err +} diff --git a/monitoring_test.go b/monitoring_test.go index 408fb6b6..83dde8da 100644 --- a/monitoring_test.go +++ b/monitoring_test.go @@ -1,6 +1,7 @@ package godo import ( + "context" "encoding/json" "fmt" "net/http" @@ -725,6 +726,89 @@ var ( }, }, } + + testLBResponseJSON = ` + { + "status": "success", + "data": { + "resultType": "matrix", + "result": [ + { + "metric": { + "lb_id": "d699d327-5ad6-4894-8172-e8105f711cd4", + "region": "s2r1" + }, + "values": [ + [ + 1729453800, + "1.5082956259357405" + ], + [ + 1729453920, + "1.4755197853656865" + ], + [ + 1729454040, + "1.2758099714636817" + ], + [ + 1729454160, + "1.4922870556695833" + ], + [ + 1729454280, + "1.509813789633041" + ], + [ + 1729454400, + "1.3252809931070697" + ] + ] + } + ] + } + }` + + testLBResponse = &MetricsResponse{ + Status: "success", + Data: MetricsData{ + ResultType: "matrix", + Result: []metrics.SampleStream{ + { + Metric: metrics.Metric{ + "lb_id": "d699d327-5ad6-4894-8172-e8105f711cd4", + "region": "s2r1", + }, + Values: []metrics.SamplePair{ + { + Timestamp: 1729453800000, + Value: 1.5082956259357405, + }, + { + Timestamp: 1729453920000, + Value: 1.4755197853656865, + }, + { + Timestamp: 1729454040000, + Value: 1.2758099714636817, + }, + { + Timestamp: 1729454160000, + Value: 1.4922870556695833, + }, + { + Timestamp: 1729454280000, + Value: 1.509813789633041, + }, + { + Timestamp: 1729454400000, + Value: 1.3252809931070697, + }, + }, + }, + }, + }, + } ) func TestAlertPolicies_List(t *testing.T) { @@ -1305,3 +1389,123 @@ func TestGetDropletCPU(t *testing.T) { assert.Equal(t, testCPUResponse, metricsResp) } + +func TestGetLoadBalancerMetrics(t *testing.T) { + setup() + defer teardown() + + for _, tc := range []struct { + testFunc func(ctx context.Context, args *LoadBalancerMetricsRequest) (*MetricsResponse, *Response, error) + path string + }{ + { + client.Monitoring.GetLoadBalancerFrontendHttpRequestsPerSecond, + "/frontend_http_requests_per_second"}, + { + client.Monitoring.GetLoadBalancerFrontendConnectionsCurrent, + "/frontend_connections_current"}, + { + client.Monitoring.GetLoadBalancerFrontendConnectionsLimit, + "/frontend_connections_limit"}, + { + client.Monitoring.GetLoadBalancerFrontendCpuUtilization, + "/frontend_cpu_utilization"}, + { + client.Monitoring.GetLoadBalancerFrontendNetworkThroughputHttp, + "/frontend_network_throughput_http"}, + { + client.Monitoring.GetLoadBalancerFrontendNetworkThroughputUdp, + "/frontend_network_throughput_udp"}, + { + client.Monitoring.GetLoadBalancerFrontendNetworkThroughputTcp, + "/frontend_network_throughput_tcp"}, + { + client.Monitoring.GetLoadBalancerFrontendNlbTcpNetworkThroughput, + "/frontend_nlb_tcp_network_throughput"}, + { + client.Monitoring.GetLoadBalancerFrontendNlbUdpNetworkThroughput, + "/frontend_nlb_udp_network_throughput"}, + { + client.Monitoring.GetLoadBalancerFrontendFirewallDroppedBytes, + "/frontend_firewall_dropped_bytes"}, + { + client.Monitoring.GetLoadBalancerFrontendFirewallDroppedPackets, + "/frontend_firewall_dropped_packets"}, + { + client.Monitoring.GetLoadBalancerFrontendHttpResponses, + "/frontend_http_responses"}, + { + client.Monitoring.GetLoadBalancerFrontendTlsConnectionsCurrent, + "/frontend_tls_connections_current"}, + { + client.Monitoring.GetLoadBalancerFrontendTlsConnectionsLimit, + "/frontend_tls_connections_limit"}, + { + client.Monitoring.GetLoadBalancerFrontendTlsConnectionsExceedingRateLimit, + "/frontend_tls_connections_exceeding_rate_limit"}, + { + client.Monitoring.GetLoadBalancerDropletsHttpSessionDurationAvg, + "/droplets_http_session_duration_avg"}, + { + client.Monitoring.GetLoadBalancerDropletsHttpSessionDuration50P, + "/droplets_http_session_duration_50p"}, + { + client.Monitoring.GetLoadBalancerDropletsHttpSessionDuration95P, + "/droplets_http_session_duration_95p"}, + { + client.Monitoring.GetLoadBalancerDropletsHttpResponseTimeAvg, + "/droplets_http_response_time_avg"}, + { + client.Monitoring.GetLoadBalancerDropletsHttpResponseTime50P, + "/droplets_http_response_time_50p"}, + { + client.Monitoring.GetLoadBalancerDropletsHttpResponseTime95P, + "/droplets_http_response_time_95p"}, + { + client.Monitoring.GetLoadBalancerDropletsHttpResponseTime99P, + "/droplets_http_response_time_99p"}, + { + client.Monitoring.GetLoadBalancerDropletsQueueSize, + "/droplets_queue_size"}, + { + client.Monitoring.GetLoadBalancerDropletsHttpResponses, + "/droplets_http_responses"}, + { + client.Monitoring.GetLoadBalancerDropletsConnections, + "/droplets_connections"}, + { + client.Monitoring.GetLoadBalancerDropletsHealthChecks, + "/droplets_health_checks"}, + { + client.Monitoring.GetLoadBalancerDropletsDowntime, + "/droplets_downtime", + }, + } { + now := time.Now() + metricReq := &LoadBalancerMetricsRequest{ + LoadBalancerID: "123", + Start: now.Add(-300 * time.Second), + End: now, + } + + mux.HandleFunc("/v2/monitoring/metrics/load_balancer"+tc.path, func(w http.ResponseWriter, r *http.Request) { + lbID := r.URL.Query().Get("lb_id") + start := r.URL.Query().Get("start") + end := r.URL.Query().Get("end") + + assert.Equal(t, metricReq.LoadBalancerID, lbID) + assert.Equal(t, fmt.Sprintf("%d", metricReq.Start.Unix()), start) + assert.Equal(t, fmt.Sprintf("%d", metricReq.End.Unix()), end) + testMethod(t, r, http.MethodGet) + + fmt.Fprintf(w, testLBResponseJSON) + }) + + metricsResp, _, err := tc.testFunc(ctx, metricReq) + if err != nil { + t.Errorf("Monitoring.%v returned error: %v", tc.path, err) + } + + assert.Equal(t, testLBResponse, metricsResp) + } +}