Skip to content

Commit 1f23496

Browse files
shlomi-noachshlomi-noach
authored
Multi-metrics throttler: adding InnoDB history_list_length metric (#17262)
Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com>
1 parent 69b381f commit 1f23496

File tree

5 files changed

+123
-1
lines changed

5 files changed

+123
-1
lines changed

‎go/test/endtoend/tabletmanager/throttler_topo/throttler_test.go

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -175,6 +175,16 @@ func throttledApps(tablet *cluster.Vttablet) (resp *http.Response, respBody stri
175175
return resp, respBody, err
176176
}
177177

178+
func vitessThrottleCheck(tablet *cluster.Vttablet, skipRequestHeartbeats bool) (*vtctldatapb.CheckThrottlerResponse, error) {
179+
flags := &throttle.CheckFlags{
180+
Scope: base.ShardScope,
181+
SkipRequestHeartbeats: skipRequestHeartbeats,
182+
MultiMetricsEnabled: true,
183+
}
184+
resp, err := throttler.CheckThrottler(clusterInstance, tablet, throttlerapp.VitessName, flags)
185+
return resp, err
186+
}
187+
178188
func throttleCheck(tablet *cluster.Vttablet, skipRequestHeartbeats bool) (*vtctldatapb.CheckThrottlerResponse, error) {
179189
flags := &throttle.CheckFlags{
180190
Scope: base.ShardScope,
@@ -305,6 +315,17 @@ func TestInitialThrottler(t *testing.T) {
305315
waitForThrottleCheckStatus(t, primaryTablet, tabletmanagerdatapb.CheckThrottlerResponseCode_THRESHOLD_EXCEEDED)
306316
})
307317
t.Run("setting high threshold", func(t *testing.T) {
318+
{
319+
req := &vtctldatapb.UpdateThrottlerConfigRequest{MetricName: base.LoadAvgMetricName.String(), Threshold: 5555}
320+
_, err := throttler.UpdateThrottlerTopoConfig(clusterInstance, req, nil, nil)
321+
assert.NoError(t, err)
322+
}
323+
{
324+
req := &vtctldatapb.UpdateThrottlerConfigRequest{MetricName: base.MysqldLoadAvgMetricName.String(), Threshold: 5555}
325+
_, err := throttler.UpdateThrottlerTopoConfig(clusterInstance, req, nil, nil)
326+
assert.NoError(t, err)
327+
}
328+
308329
req := &vtctldatapb.UpdateThrottlerConfigRequest{Threshold: extremelyHighThreshold.Seconds()}
309330
_, err := throttler.UpdateThrottlerTopoConfig(clusterInstance, req, nil, nil)
310331
assert.NoError(t, err)
@@ -317,6 +338,19 @@ func TestInitialThrottler(t *testing.T) {
317338
t.Run("validating OK response from throttler with high threshold", func(t *testing.T) {
318339
waitForThrottleCheckStatus(t, primaryTablet, tabletmanagerdatapb.CheckThrottlerResponseCode_OK)
319340
})
341+
t.Run("validating vitess app throttler check", func(t *testing.T) {
342+
resp, err := vitessThrottleCheck(primaryTablet, true)
343+
require.NoError(t, err)
344+
for _, metricName := range base.KnownMetricNames {
345+
t.Run(metricName.String(), func(t *testing.T) {
346+
assert.Contains(t, resp.Check.Metrics, metricName.String())
347+
metric := resp.Check.Metrics[metricName.String()]
348+
require.NotNil(t, metric)
349+
assert.Equal(t, tabletmanagerdatapb.CheckThrottlerResponseCode_OK, metric.ResponseCode, "metric: %+v", metric)
350+
})
351+
}
352+
})
353+
320354
t.Run("setting low threshold", func(t *testing.T) {
321355
req := &vtctldatapb.UpdateThrottlerConfigRequest{Threshold: throttler.DefaultThreshold.Seconds()}
322356
_, err := throttler.UpdateThrottlerTopoConfig(clusterInstance, req, nil, nil)

‎go/vt/vttablet/tabletserver/throttle/base/metric_name.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ const (
6565
ThreadsRunningMetricName MetricName = "threads_running"
6666
CustomMetricName MetricName = "custom"
6767
LoadAvgMetricName MetricName = "loadavg"
68+
HistoryListLengthMetricName MetricName = "history_list_length"
6869
MysqldLoadAvgMetricName MetricName = "mysqld-loadavg"
6970
MysqldDatadirUsedRatioMetricName MetricName = "mysqld-datadir-used-ratio"
7071
)

‎go/vt/vttablet/tabletserver/throttle/base/metric_name_test.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -241,15 +241,17 @@ func TestKnownMetricNames(t *testing.T) {
241241
assert.Contains(t, KnownMetricNames, LoadAvgMetricName)
242242
assert.Contains(t, KnownMetricNames, CustomMetricName)
243243
assert.Contains(t, KnownMetricNames, DefaultMetricName)
244+
assert.Contains(t, KnownMetricNames, HistoryListLengthMetricName)
244245
assert.Contains(t, KnownMetricNames, MysqldLoadAvgMetricName)
245246
assert.Contains(t, KnownMetricNames, MysqldDatadirUsedRatioMetricName)
246247
}
247248

248-
func TestSingleWordCamelKnownMetricNames(t *testing.T) {
249+
func TestKnownMetricNamesPascalCase(t *testing.T) {
249250
expectCases := map[MetricName]string{
250251
LagMetricName: "Lag",
251252
ThreadsRunningMetricName: "ThreadsRunning",
252253
LoadAvgMetricName: "Loadavg",
254+
HistoryListLengthMetricName: "HistoryListLength",
253255
CustomMetricName: "Custom",
254256
DefaultMetricName: "Default",
255257
MysqldLoadAvgMetricName: "MysqldLoadavg",

‎go/vt/vttablet/tabletserver/throttle/base/self_metric_innodb_history_list_length.go

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
/*
2+
Copyright 2024 The Vitess Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package base
18+
19+
import (
20+
"context"
21+
"math"
22+
"sync/atomic"
23+
"time"
24+
)
25+
26+
var (
27+
historyListLengthQuery = "select count as history_len from information_schema.INNODB_METRICS where name = 'trx_rseg_history_len'"
28+
29+
cachedHistoryListLengthMetric atomic.Pointer[ThrottleMetric]
30+
historyListLengthCacheDuration = 5 * time.Second
31+
historyListLengthDefaultThreshold = math.Pow10(9)
32+
)
33+
34+
var _ SelfMetric = registerSelfMetric(&HistoryListLengthSelfMetric{})
35+
36+
type HistoryListLengthSelfMetric struct {
37+
}
38+
39+
func (m *HistoryListLengthSelfMetric) Name() MetricName {
40+
return HistoryListLengthMetricName
41+
}
42+
43+
func (m *HistoryListLengthSelfMetric) DefaultScope() Scope {
44+
return SelfScope
45+
}
46+
47+
func (m *HistoryListLengthSelfMetric) DefaultThreshold() float64 {
48+
return historyListLengthDefaultThreshold
49+
}
50+
51+
func (m *HistoryListLengthSelfMetric) RequiresConn() bool {
52+
return true
53+
}
54+
55+
func (m *HistoryListLengthSelfMetric) Read(ctx context.Context, params *SelfMetricReadParams) *ThrottleMetric {
56+
// This function will be called sequentially, and therefore does not need strong mutex protection. Still, we use atomics
57+
// to ensure correctness in case an external goroutine tries to read the metric concurrently.
58+
metric := cachedHistoryListLengthMetric.Load()
59+
if metric != nil {
60+
return metric
61+
}
62+
metric = ReadSelfMySQLThrottleMetric(ctx, params.Conn, historyListLengthQuery)
63+
cachedHistoryListLengthMetric.Store(metric)
64+
time.AfterFunc(historyListLengthCacheDuration, func() {
65+
cachedHistoryListLengthMetric.Store(nil)
66+
})
67+
return metric
68+
}

‎go/vt/vttablet/tabletserver/throttle/throttler_test.go

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,12 @@ var (
7171
Value: 2.718,
7272
Err: nil,
7373
},
74+
base.HistoryListLengthMetricName: &base.ThrottleMetric{
75+
Scope: base.SelfScope,
76+
Alias: "",
77+
Value: 5,
78+
Err: nil,
79+
},
7480
base.MysqldLoadAvgMetricName: &base.ThrottleMetric{
7581
Scope: base.SelfScope,
7682
Alias: "",
@@ -105,6 +111,11 @@ var (
105111
ResponseCode: tabletmanagerdatapb.CheckThrottlerResponseCode_OK,
106112
Value: 5.1,
107113
},
114+
base.HistoryListLengthMetricName.String(): {
115+
StatusCode: http.StatusOK,
116+
ResponseCode: tabletmanagerdatapb.CheckThrottlerResponseCode_OK,
117+
Value: 6,
118+
},
108119
base.MysqldLoadAvgMetricName.String(): {
109120
StatusCode: http.StatusOK,
110121
ResponseCode: tabletmanagerdatapb.CheckThrottlerResponseCode_OK,
@@ -1853,6 +1864,7 @@ func TestChecks(t *testing.T) {
18531864
assert.EqualValues(t, 26, checkResult.Metrics[base.ThreadsRunningMetricName.String()].Value) // self value, because flags.Scope is set
18541865
assert.EqualValues(t, 17, checkResult.Metrics[base.CustomMetricName.String()].Value) // self value, because flags.Scope is set
18551866
assert.EqualValues(t, 2.718, checkResult.Metrics[base.LoadAvgMetricName.String()].Value) // self value, because flags.Scope is set
1867+
assert.EqualValues(t, 5, checkResult.Metrics[base.HistoryListLengthMetricName.String()].Value) // self value, because flags.Scope is set
18561868
assert.EqualValues(t, 0.3311, checkResult.Metrics[base.MysqldLoadAvgMetricName.String()].Value) // self value, because flags.Scope is set
18571869
assert.EqualValues(t, 0.85, checkResult.Metrics[base.MysqldDatadirUsedRatioMetricName.String()].Value) // self value, because flags.Scope is set
18581870
for _, metric := range checkResult.Metrics {
@@ -1914,6 +1926,7 @@ func TestChecks(t *testing.T) {
19141926
assert.EqualValues(t, 26, checkResult.Metrics[base.ThreadsRunningMetricName.String()].Value) // shard value, because flags.Scope is set
19151927
assert.EqualValues(t, 17, checkResult.Metrics[base.CustomMetricName.String()].Value) // shard value, because flags.Scope is set
19161928
assert.EqualValues(t, 5.1, checkResult.Metrics[base.LoadAvgMetricName.String()].Value) // shard value, because flags.Scope is set
1929+
assert.EqualValues(t, 6, checkResult.Metrics[base.HistoryListLengthMetricName.String()].Value) // shard value, because flags.Scope is set
19171930
assert.EqualValues(t, 0.3311, checkResult.Metrics[base.MysqldLoadAvgMetricName.String()].Value) // shard value, because flags.Scope is set
19181931
assert.EqualValues(t, 0.87, checkResult.Metrics[base.MysqldDatadirUsedRatioMetricName.String()].Value) // shard value, because flags.Scope is set
19191932
for _, metric := range checkResult.Metrics {
@@ -1948,6 +1961,7 @@ func TestChecks(t *testing.T) {
19481961
assert.EqualValues(t, 26, checkResult.Metrics[base.ThreadsRunningMetricName.String()].Value) // self value, because "self" is the default scope for threads_running
19491962
assert.EqualValues(t, 17, checkResult.Metrics[base.CustomMetricName.String()].Value) // self value, because "self" is the default scope for custom
19501963
assert.EqualValues(t, 2.718, checkResult.Metrics[base.LoadAvgMetricName.String()].Value) // self value, because "self" is the default scope for loadavg
1964+
assert.EqualValues(t, 5, checkResult.Metrics[base.HistoryListLengthMetricName.String()].Value) // self value, because "self" is the default scope for loadavg
19511965
assert.EqualValues(t, 0.3311, checkResult.Metrics[base.MysqldLoadAvgMetricName.String()].Value) // self value, because "self" is the default scope for loadavg
19521966
assert.EqualValues(t, 0.85, checkResult.Metrics[base.MysqldDatadirUsedRatioMetricName.String()].Value) // self value, because "self" is the default scope for loadavg
19531967
assert.EqualValues(t, base.ShardScope.String(), checkResult.Metrics[base.LagMetricName.String()].Scope)
@@ -1970,6 +1984,7 @@ func TestChecks(t *testing.T) {
19701984
base.MetricName("custom"),
19711985
base.MetricName("shard/loadavg"),
19721986
base.MetricName("shard/mysqld-loadavg"),
1987+
base.MetricName("self/history_list_length"),
19731988
base.MetricName("self/mysqld-datadir-used-ratio"),
19741989
base.MetricName("default"),
19751990
}
@@ -1986,6 +2001,7 @@ func TestChecks(t *testing.T) {
19862001
assert.EqualValues(t, 26, checkResult.Metrics[base.ThreadsRunningMetricName.String()].Value) // shard value, even though scope name is in metric name
19872002
assert.EqualValues(t, 17, checkResult.Metrics[base.CustomMetricName.String()].Value) // shard value because flags.Scope is set
19882003
assert.EqualValues(t, 5.1, checkResult.Metrics[base.LoadAvgMetricName.String()].Value) // shard value, not because scope name is in metric name but because flags.Scope is set
2004+
assert.EqualValues(t, 6, checkResult.Metrics[base.HistoryListLengthMetricName.String()].Value) // shard value, even though scope name is in metric name
19892005
assert.EqualValues(t, 0.3311, checkResult.Metrics[base.MysqldLoadAvgMetricName.String()].Value) // shard value, not because scope name is in metric name but because flags.Scope is set
19902006
assert.EqualValues(t, 0.87, checkResult.Metrics[base.MysqldDatadirUsedRatioMetricName.String()].Value) // shard value, even though scope name is in metric name
19912007
for _, metric := range checkResult.Metrics {
@@ -2257,6 +2273,7 @@ func TestReplica(t *testing.T) {
22572273
assert.Error(t, metricResult.Error, "metricName=%v, value=%v, threshold=%v", metricName, metricResult.Value, metricResult.Threshold)
22582274
assert.ErrorIs(t, metricResult.Error, base.ErrThresholdExceeded)
22592275
case base.ThreadsRunningMetricName,
2276+
base.HistoryListLengthMetricName,
22602277
base.MysqldLoadAvgMetricName,
22612278
base.MysqldDatadirUsedRatioMetricName:
22622279
assert.NoError(t, metricResult.Error, "metricName=%v, value=%v, threshold=%v", metricName, metricResult.Value, metricResult.Threshold)

0 commit comments

Comments
 (0)