Skip to content

Commit b7e34ea

Browse files
jianlinliujianlinliu
committed
extract Operator Progressing / Degraded Counts and Timing
1 parent 5d46d82 commit b7e34ea

File tree

2 files changed

+276
-0
lines changed

2 files changed

+276
-0
lines changed

‎pkg/monitortests/clusterversionoperator/operatorstateanalyzer/monitortest.go‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,15 @@ package operatorstateanalyzer
22

33
import (
44
"context"
5+
"fmt"
6+
"path/filepath"
7+
"sort"
58
"time"
69

710
"github.com/openshift/origin/pkg/monitortestframework"
11+
"github.com/sirupsen/logrus"
812

13+
"github.com/openshift/origin/pkg/dataloader"
914
"github.com/openshift/origin/pkg/monitor/monitorapi"
1015
"github.com/openshift/origin/pkg/test/ginkgo/junitapi"
1116
"k8s.io/client-go/rest"
@@ -14,6 +19,16 @@ import (
1419
type operatorStateChecker struct {
1520
}
1621

22+
type OperatorStateMetrics struct {
23+
OperatorName string
24+
ProgressingCount int
25+
TotalProgressingSeconds float64
26+
MaxIndividualProgressingSeconds float64
27+
DegradedCount int
28+
TotalDegradedSeconds float64
29+
MaxIndividualDegradedSeconds float64
30+
}
31+
1732
func NewAnalyzer() monitortestframework.MonitorTest {
1833
return &operatorStateChecker{}
1934
}
@@ -44,9 +59,102 @@ func (*operatorStateChecker) EvaluateTestsFromConstructedIntervals(ctx context.C
4459
}
4560

4661
func (*operatorStateChecker) WriteContentToStorage(ctx context.Context, storageDir, timeSuffix string, finalIntervals monitorapi.Intervals, finalResourceState monitorapi.ResourcesMap) error {
62+
metrics := calculateOperatorStateMetrics(finalIntervals)
63+
if len(metrics) > 0 {
64+
rows := generateRowsFromMetrics(metrics)
65+
dataFile := dataloader.DataFile{
66+
TableName: "operator_state_metrics",
67+
Schema: map[string]dataloader.DataType{
68+
"Operator": dataloader.DataTypeString,
69+
"State": dataloader.DataTypeString,
70+
"Count": dataloader.DataTypeFloat64,
71+
"TotalSeconds": dataloader.DataTypeFloat64,
72+
"MaxIndividualDurationSeconds": dataloader.DataTypeFloat64,
73+
},
74+
Rows: rows,
75+
}
76+
fileName := filepath.Join(storageDir, fmt.Sprintf("operator-state-metrics%s-%s", timeSuffix, dataloader.AutoDataLoaderSuffix))
77+
if err := dataloader.WriteDataFile(fileName, dataFile); err != nil {
78+
return fmt.Errorf("failed to write operator state metrics: %w", err)
79+
}
80+
logrus.Infof("Write operator state metrics to %s successfully.", fileName)
81+
}
82+
4783
return nil
4884
}
4985

86+
// calculateOperatorStateMetrics processes raw intervals and aggregates them into a metrics summary map.
87+
func calculateOperatorStateMetrics(finalIntervals monitorapi.Intervals) map[string]*OperatorStateMetrics {
88+
metrics := make(map[string]*OperatorStateMetrics)
89+
90+
for _, interval := range finalIntervals {
91+
if interval.Source != monitorapi.SourceOperatorState {
92+
continue
93+
}
94+
if interval.Locator.Type != monitorapi.LocatorTypeClusterOperator {
95+
continue
96+
}
97+
operatorName := interval.Locator.Keys[monitorapi.LocatorClusterOperatorKey]
98+
if _, ok := metrics[operatorName]; !ok {
99+
metrics[operatorName] = &OperatorStateMetrics{OperatorName: operatorName}
100+
}
101+
102+
duration := interval.To.Sub(interval.From).Seconds()
103+
condition := interval.Message.Annotations[monitorapi.AnnotationCondition]
104+
105+
switch condition {
106+
case "Progressing":
107+
metrics[operatorName].ProgressingCount++
108+
metrics[operatorName].TotalProgressingSeconds += duration
109+
if duration > metrics[operatorName].MaxIndividualProgressingSeconds {
110+
metrics[operatorName].MaxIndividualProgressingSeconds = duration
111+
}
112+
case "Degraded":
113+
metrics[operatorName].DegradedCount++
114+
metrics[operatorName].TotalDegradedSeconds += duration
115+
if duration > metrics[operatorName].MaxIndividualDegradedSeconds {
116+
metrics[operatorName].MaxIndividualDegradedSeconds = duration
117+
}
118+
}
119+
}
120+
return metrics
121+
}
122+
123+
// generateRowsFromMetrics converts the aggregated metrics map into a slice of rows for the dataloader.
124+
func generateRowsFromMetrics(metrics map[string]*OperatorStateMetrics) []map[string]string {
125+
rows := []map[string]string{}
126+
127+
// Sort operator names for consistent output order in tests
128+
operatorNames := make([]string, 0, len(metrics))
129+
for name := range metrics {
130+
operatorNames = append(operatorNames, name)
131+
}
132+
sort.Strings(operatorNames)
133+
134+
for _, operatorName := range operatorNames {
135+
metric := metrics[operatorName]
136+
if metric.ProgressingCount > 0 {
137+
rows = append(rows, map[string]string{
138+
"Operator": operatorName,
139+
"State": "Progressing",
140+
"Count": fmt.Sprintf("%f", float64(metric.ProgressingCount)),
141+
"TotalSeconds": fmt.Sprintf("%f", metric.TotalProgressingSeconds),
142+
"MaxIndividualDurationSeconds": fmt.Sprintf("%f", metric.MaxIndividualProgressingSeconds),
143+
})
144+
}
145+
if metric.DegradedCount > 0 {
146+
rows = append(rows, map[string]string{
147+
"Operator": operatorName,
148+
"State": "Degraded",
149+
"Count": fmt.Sprintf("%f", float64(metric.DegradedCount)),
150+
"TotalSeconds": fmt.Sprintf("%f", metric.TotalDegradedSeconds),
151+
"MaxIndividualDurationSeconds": fmt.Sprintf("%f", metric.MaxIndividualDegradedSeconds),
152+
})
153+
}
154+
}
155+
return rows
156+
}
157+
50158
func (*operatorStateChecker) Cleanup(ctx context.Context) error {
51159
// TODO wire up the start to a context we can kill here
52160
return nil

‎pkg/monitortests/clusterversionoperator/operatorstateanalyzer/monitortest_test.go‎

Lines changed: 168 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,168 @@
1+
package operatorstateanalyzer
2+
3+
import (
4+
"testing"
5+
"time"
6+
7+
"github.com/openshift/origin/pkg/monitor/monitorapi"
8+
"github.com/stretchr/testify/assert"
9+
"github.com/stretchr/testify/require"
10+
)
11+
12+
func TestOperatorStateAnalyzer(t *testing.T) {
13+
tests := []struct {
14+
name string
15+
intervals monitorapi.Intervals
16+
expectedMetrics map[string]*OperatorStateMetrics
17+
expectedRows []map[string]string
18+
}{
19+
{
20+
name: "single operator, progressing and degraded",
21+
intervals: monitorapi.Intervals{
22+
makeTestInterval("operator-a", "Progressing", 10),
23+
makeTestInterval("operator-a", "Progressing", 5),
24+
makeTestInterval("operator-a", "Degraded", 15),
25+
},
26+
expectedMetrics: map[string]*OperatorStateMetrics{
27+
"operator-a": {
28+
OperatorName: "operator-a",
29+
ProgressingCount: 2,
30+
TotalProgressingSeconds: 15,
31+
MaxIndividualProgressingSeconds: 10,
32+
DegradedCount: 1,
33+
TotalDegradedSeconds: 15,
34+
MaxIndividualDegradedSeconds: 15,
35+
},
36+
},
37+
expectedRows: []map[string]string{
38+
{
39+
"Operator": "operator-a",
40+
"State": "Progressing",
41+
"Count": "2.000000",
42+
"TotalSeconds": "15.000000",
43+
"MaxIndividualDurationSeconds": "10.000000",
44+
},
45+
{
46+
"Operator": "operator-a",
47+
"State": "Degraded",
48+
"Count": "1.000000",
49+
"TotalSeconds": "15.000000",
50+
"MaxIndividualDurationSeconds": "15.000000",
51+
},
52+
},
53+
},
54+
{
55+
name: "multiple operators",
56+
intervals: monitorapi.Intervals{
57+
makeTestInterval("operator-a", "Progressing", 10),
58+
makeTestInterval("operator-b", "Degraded", 20),
59+
},
60+
expectedMetrics: map[string]*OperatorStateMetrics{
61+
"operator-a": {
62+
OperatorName: "operator-a",
63+
ProgressingCount: 1,
64+
TotalProgressingSeconds: 10,
65+
MaxIndividualProgressingSeconds: 10,
66+
},
67+
"operator-b": {
68+
OperatorName: "operator-b",
69+
DegradedCount: 1,
70+
TotalDegradedSeconds: 20,
71+
MaxIndividualDegradedSeconds: 20,
72+
},
73+
},
74+
expectedRows: []map[string]string{
75+
{
76+
"Operator": "operator-a",
77+
"State": "Progressing",
78+
"Count": "1.000000",
79+
"TotalSeconds": "10.000000",
80+
"MaxIndividualDurationSeconds": "10.000000",
81+
},
82+
{
83+
"Operator": "operator-b",
84+
"State": "Degraded",
85+
"Count": "1.000000",
86+
"TotalSeconds": "20.000000",
87+
"MaxIndividualDurationSeconds": "20.000000",
88+
},
89+
},
90+
},
91+
{
92+
name: "no relevant intervals",
93+
intervals: monitorapi.Intervals{},
94+
expectedMetrics: map[string]*OperatorStateMetrics{},
95+
expectedRows: []map[string]string{},
96+
},
97+
{
98+
name: "operator with only degraded state",
99+
intervals: monitorapi.Intervals{
100+
makeTestInterval("operator-c", "Degraded", 30),
101+
},
102+
expectedMetrics: map[string]*OperatorStateMetrics{
103+
"operator-c": {
104+
OperatorName: "operator-c",
105+
DegradedCount: 1,
106+
TotalDegradedSeconds: 30,
107+
MaxIndividualDegradedSeconds: 30,
108+
},
109+
},
110+
expectedRows: []map[string]string{
111+
{
112+
"Operator": "operator-c",
113+
"State": "Degraded",
114+
"Count": "1.000000",
115+
"TotalSeconds": "30.000000",
116+
"MaxIndividualDurationSeconds": "30.000000",
117+
},
118+
},
119+
},
120+
}
121+
122+
for _, tc := range tests {
123+
t.Run(tc.name, func(t *testing.T) {
124+
// Test calculateOperatorStateMetrics
125+
metrics := calculateOperatorStateMetrics(tc.intervals)
126+
require.Equal(t, len(tc.expectedMetrics), len(metrics), "number of operators should match")
127+
for op, expected := range tc.expectedMetrics {
128+
actual, ok := metrics[op]
129+
require.True(t, ok, "operator %s not found in metrics", op)
130+
assert.Equal(t, expected.OperatorName, actual.OperatorName, "OperatorName should match")
131+
assert.Equal(t, expected.ProgressingCount, actual.ProgressingCount, "ProgressingCount should match")
132+
assert.InDelta(t, expected.TotalProgressingSeconds, actual.TotalProgressingSeconds, 0.001, "TotalProgressingSeconds should match")
133+
assert.InDelta(t, expected.MaxIndividualProgressingSeconds, actual.MaxIndividualProgressingSeconds, 0.001, "MaxIndividualProgressingSeconds should match")
134+
assert.Equal(t, expected.DegradedCount, actual.DegradedCount, "DegradedCount should match")
135+
assert.InDelta(t, expected.TotalDegradedSeconds, actual.TotalDegradedSeconds, 0.001, "TotalDegradedSeconds should match")
136+
assert.InDelta(t, expected.MaxIndividualDegradedSeconds, actual.MaxIndividualDegradedSeconds, 0.001, "MaxIndividualDegradedSeconds should match")
137+
}
138+
139+
// Test generateRowsFromMetrics
140+
rows := generateRowsFromMetrics(metrics)
141+
assert.ElementsMatch(t, tc.expectedRows, rows, "generated rows should match expected rows")
142+
})
143+
}
144+
}
145+
146+
// Helper function to create intervals for testing
147+
func makeTestInterval(operatorName, condition string, durationSeconds float64) monitorapi.Interval {
148+
from := time.Unix(1, 0)
149+
to := from.Add(time.Duration(durationSeconds * float64(time.Second)))
150+
return monitorapi.Interval{
151+
Source: monitorapi.SourceOperatorState,
152+
Condition: monitorapi.Condition{
153+
Locator: monitorapi.Locator{
154+
Type: monitorapi.LocatorTypeClusterOperator,
155+
Keys: map[monitorapi.LocatorKey]string{
156+
monitorapi.LocatorClusterOperatorKey: operatorName,
157+
},
158+
},
159+
Message: monitorapi.Message{
160+
Annotations: map[monitorapi.AnnotationKey]string{
161+
monitorapi.AnnotationCondition: condition,
162+
},
163+
},
164+
},
165+
From: from,
166+
To: to,
167+
}
168+
}

0 commit comments

Comments
 (0)